Skip to content

Commit f87ae6b

Browse files
moprashantCopilot
moprashant
and
Copilot
authored
Add Azure Linux support to MongoDB/YCSB workload (declarative SupportedLinuxDistributions filtering) (#784)
* Add Azure Linux support to the MongoDB/YCSB workload Extends PERF-MONGODB-YCSB.json to run on Azure Linux in addition to Ubuntu, and introduces a general-purpose mechanism for scoping any profile component to specific Linux distributions. SupportedLinuxDistributions component parameter Components may now declare "SupportedLinuxDistributions": "AzureLinux,Ubuntu". IsSupported() evaluates it after the existing SupportedPlatforms check, and a component that declares it never runs on non-Linux systems. Profile The MongoDB installation steps are split into an apt path scoped to Debian/Ubuntu and a dnf path scoped to AzureLinux. lshw is added to the package prerequisites; VirtualClient uses it for disk discovery and it is not present by default on Azure Linux. gnupg is corrected to gnupg2 for the dnf/yum package names. MongoDBServerExecutor The mongod service account is resolved at runtime rather than hardcoded to mongodb. RPM packages create mongod while Debian packages create mongodb, so the data directory was left owned by root on Azure Linux and mongod exited with status 100. The MongoDB port is opened via IFirewallManager during initialization. Azure Linux applies a default-deny policy to inbound traffic, so the client could not reach the server and runs completed with zero recorded operations. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix duplicate key insert failures in YCSB run-phase insert scenarios Read_Latest (workloadd) and Short_Range_Scan (workloade) both perform 5% run-phase inserts. YCSB seeds the insert key sequence for both scenarios at recordcount, so the scenario that runs second collides with keys already written by the first and every insert fails with: E11000 duplicate key error collection: ycsb.usertable index: _id_ Measured on a 2500000 record run, Short_Range_Scan reported INSERT-Operations=0 with INSERT-FAILED-Operations=3445, matching the 3445 duplicate key errors in the logs exactly. Setting mongodb.upsert=true makes colliding inserts update the existing document instead of failing. This is the documented behaviour of the YCSB MongoDB binding for partially loaded data sets and requires no change to RecordCount, operation mix, metric names or scenario names. This issue is not platform specific and reproduces on both Ubuntu and Azure Linux. Verified on Azure Linux 3 and Ubuntu 24.04 (2 VM client/server, exit code 0, all 7 scenarios): Short_Range_Scan now reports non-zero INSERT-Operations with zero failed operations and no E11000 errors. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Document Azure Linux support for the MongoDB YCSB profile The profile metadata declares SupportedOperatingSystems as 'AzureLinux,Ubuntu', but the workload documentation still listed Ubuntu only. This aligns the documentation with the profile. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix line endings on the MongoDB docs update The previous commit was uploaded from a CRLF working copy, which rewrote every line. This restores LF endings so the change is the intended single line. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent af6d6cf commit f87ae6b

6 files changed

Lines changed: 325 additions & 9 deletions

File tree

‎src/VirtualClient/VirtualClient.Actions.UnitTests/MongoDB/MongoDBServerExecutorTests.cs‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -342,6 +342,77 @@ public async Task MongoDBServerExecutor_InitializeAsync_WithDiskFilter_CallsInit
342342
"ServerApiClient should be initialized");
343343
}
344344

345+
[Test]
346+
public async Task MongoDBServerExecutor_ConfigureDisk_ResolvesTheMongoDBServiceUserForThePlatform()
347+
{
348+
// SETUP: A DiskFilter triggers the disk configuration workflow.
349+
this.mockFixture.Parameters["DiskFilter"] = "BiggestSize";
350+
this.mockFixture.Parameters["DiskDevicePath"] = "/dev/nvme0n1";
351+
352+
var volumes = new List<DiskVolume>();
353+
var disk = new Disk(index: 0, devicePath: "/dev/nvme0n1", volumes: volumes, properties: null);
354+
this.mockFixture.DiskManager.Setup(dm => dm.GetDisksAsync(It.IsAny<CancellationToken>()))
355+
.ReturnsAsync(new List<Disk> { disk });
356+
357+
List<string> commandsExecuted = new List<string>();
358+
this.mockFixture.ProcessManager.OnCreateProcess = (exe, args, workingDir) =>
359+
{
360+
commandsExecuted.Add($"{exe} {args}");
361+
this.mockFixture.Process.StandardOutput.Clear();
362+
this.mockFixture.Process.StandardOutput.Append("{ \"ok\" : 1 }");
363+
this.mockFixture.Process.ExitCode = 0;
364+
return this.mockFixture.Process;
365+
};
366+
367+
var executor = new TestableMongoDBServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters);
368+
369+
// ACT
370+
await executor.InitializeAsync(EventContext.Persisted(), CancellationToken.None);
371+
372+
// ASSERT: The MongoDB service account is named 'mongodb' by Debian/Ubuntu packages but
373+
// 'mongod' by RPM packages (Azure Linux, RHEL, Fedora). Hardcoding either one leaves the
374+
// data directory owned by root on the other, and mongod then fails to create its journal.
375+
string chownCommand = commandsExecuted.FirstOrDefault(cmd => cmd.Contains("chown", StringComparison.OrdinalIgnoreCase));
376+
377+
Assert.IsNotNull(chownCommand, "The data directory ownership command should have been executed.");
378+
379+
Assert.IsFalse(
380+
chownCommand.Contains("chown -R mongodb:mongodb", StringComparison.OrdinalIgnoreCase),
381+
"The ownership command must not hardcode the Debian-only 'mongodb' account.");
382+
383+
Assert.IsTrue(
384+
chownCommand.Contains("id -u mongod", StringComparison.OrdinalIgnoreCase),
385+
"The ownership command should probe for the RPM 'mongod' account.");
386+
387+
Assert.IsTrue(
388+
chownCommand.Contains("echo mongodb", StringComparison.OrdinalIgnoreCase),
389+
"The ownership command should fall back to the Debian 'mongodb' account.");
390+
}
391+
392+
[Test]
393+
public async Task MongoDBServerExecutor_InitializeAsync_OpensTheMongoDBPortOnTheLocalFirewall()
394+
{
395+
// SETUP: Capture the firewall entries the executor asks to be opened.
396+
List<FirewallEntry> firewallEntries = new List<FirewallEntry>();
397+
this.mockFixture.FirewallManager
398+
.Setup(mgr => mgr.EnableInboundConnectionsAsync(It.IsAny<IEnumerable<FirewallEntry>>(), It.IsAny<CancellationToken>()))
399+
.Callback<IEnumerable<FirewallEntry>, CancellationToken>((entries, token) => firewallEntries.AddRange(entries))
400+
.Returns(Task.CompletedTask);
401+
402+
this.mockFixture.Parameters["Port"] = 27017;
403+
404+
var executor = new TestableMongoDBServerExecutor(this.mockFixture.Dependencies, this.mockFixture.Parameters);
405+
406+
// ACT
407+
await executor.InitializeAsync(EventContext.Persisted(), CancellationToken.None);
408+
409+
// ASSERT: Distros such as Azure Linux apply a default-deny inbound policy. Without opening
410+
// the port, the YCSB client times out connecting to the server and loads zero records.
411+
Assert.AreEqual(1, firewallEntries.Count, "The MongoDB port should have been opened on the local firewall.");
412+
Assert.AreEqual("tcp", firewallEntries[0].Protocol);
413+
CollectionAssert.AreEqual(new List<int> { 27017 }, firewallEntries[0].Ports.ToList());
414+
}
415+
345416
[Test]
346417
public async Task MongoDBServerExecutor_InitializeAsync_CallsConfigureBindAddressAndStartServer()
347418
{

‎src/VirtualClient/VirtualClient.Actions/MongoDB/MongoDBServerExecutor.cs‎

Lines changed: 40 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,16 @@ namespace VirtualClient.Actions
2424
[SupportedPlatforms("linux-arm64,linux-x64")]
2525
public class MongoDBServerExecutor : MongoDBExecutor
2626
{
27+
/// <summary>
28+
/// The MongoDB service account created by RPM-based packages (Azure Linux, RHEL, Fedora).
29+
/// </summary>
30+
private const string RpmServiceUser = "mongod";
31+
32+
/// <summary>
33+
/// The MongoDB service account created by Debian-based packages (Ubuntu, Debian).
34+
/// </summary>
35+
private const string DebianServiceUser = "mongodb";
36+
2737
private IFileSystem fileSystem;
2838
private ISystemManagement systemManagement;
2939
private bool disposed;
@@ -68,6 +78,9 @@ protected override async Task InitializeAsync(EventContext telemetryContext, Can
6878

6979
this.InitializeApiClients();
7080

81+
await MongoDBServerExecutor.OpenFirewallPortsAsync(this.Port, this.systemManagement.FirewallManager, cancellationToken)
82+
.ConfigureAwait(false);
83+
7184
// Initialize disk if DiskFilter is specified
7285
if (!string.IsNullOrWhiteSpace(this.DiskFilter))
7386
{
@@ -122,6 +135,25 @@ protected override void Dispose(bool disposing)
122135
}
123136
}
124137

138+
/// <summary>
139+
/// Opens the MongoDB port on the local firewall so that the client instance is able to
140+
/// connect to the MongoDB server. Distros such as Azure Linux apply a default-deny policy
141+
/// to inbound traffic, so the port must be opened explicitly.
142+
/// </summary>
143+
private static Task OpenFirewallPortsAsync(int port, IFirewallManager firewallManager, CancellationToken cancellationToken)
144+
{
145+
return firewallManager.EnableInboundConnectionsAsync(
146+
new List<FirewallEntry>
147+
{
148+
new FirewallEntry(
149+
"MongoDB: Allow Multiple Machines communications",
150+
"Allows individual machine instances to communicate with other machine in client-server scenario",
151+
"tcp",
152+
new List<int> { port })
153+
},
154+
cancellationToken);
155+
}
156+
125157
/// <summary>
126158
/// Configures MongoDB to bind to all network interfaces.
127159
/// </summary>
@@ -266,10 +298,16 @@ await this.ExecuteMongoDBCommandAsync(
266298
telemetryContext,
267299
cancellationToken).ConfigureAwait(false);
268300

269-
// Set permissions
301+
// Set permissions. The MongoDB service account name differs by package format:
302+
// Debian/Ubuntu packages create 'mongodb' whereas RPM-based distributions
303+
// (Azure Linux, RHEL, Fedora) create 'mongod'. Resolve it at runtime so the
304+
// data directory is owned by the account mongod actually runs as.
305+
string resolveServiceUser = $"MONGO_USER=$(id -u {MongoDBServerExecutor.RpmServiceUser} >/dev/null 2>&1 && echo {MongoDBServerExecutor.RpmServiceUser} || echo {MongoDBServerExecutor.DebianServiceUser}); " +
306+
$"sudo chown -R $MONGO_USER:$MONGO_USER {mongoDataPath}";
307+
270308
await this.ExecuteMongoDBCommandAsync(
271309
"bash",
272-
$"-c \"sudo chown -R mongodb:mongodb {mongoDataPath}\"",
310+
$"-c \"{resolveServiceUser}\"",
273311
"SetPermissions",
274312
telemetryContext,
275313
cancellationToken).ConfigureAwait(false);

‎src/VirtualClient/VirtualClient.Contracts.UnitTests/VirtualClientComponentTests.cs‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -724,6 +724,121 @@ public void VirtualClientComponentIsSupportedRespectsSupportedPlatformAttribute(
724724
Assert.IsFalse(VirtualClientComponent.IsSupported(component));
725725
}
726726

727+
[Test]
728+
public void VirtualClientComponentSupportedLinuxDistributionsIsEmptyWhenTheParameterIsNotDefined()
729+
{
730+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
731+
732+
Assert.IsNotNull(component.SupportedLinuxDistributions);
733+
Assert.IsEmpty(component.SupportedLinuxDistributions);
734+
}
735+
736+
[Test]
737+
public void VirtualClientComponentSupportedLinuxDistributionsParsesDelimitedValues()
738+
{
739+
this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "Debian,Ubuntu";
740+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
741+
742+
CollectionAssert.AreEqual(new List<string> { "Debian", "Ubuntu" }, component.SupportedLinuxDistributions);
743+
}
744+
745+
[Test]
746+
public void VirtualClientComponentIsSupportedWhenTheSupportedLinuxDistributionsParameterIsNotDefined()
747+
{
748+
// A component that does not define the parameter is not filtered on the Linux distribution
749+
// at all and thus executes on any distribution.
750+
this.mockFixture.Setup(PlatformID.Unix);
751+
this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora);
752+
753+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
754+
755+
Assert.IsTrue(component.IsSupported());
756+
}
757+
758+
[Test]
759+
[TestCase("AzureLinux")]
760+
[TestCase("Debian,Ubuntu,AzureLinux")]
761+
[TestCase("azurelinux")]
762+
[TestCase("AZURELINUX")]
763+
[TestCase(" AzureLinux , Ubuntu ")]
764+
public void VirtualClientComponentIsSupportedWhenTheLinuxDistributionMatchesTheSupportedLinuxDistributions(string supportedDistributions)
765+
{
766+
this.mockFixture.Setup(PlatformID.Unix);
767+
this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora);
768+
this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = supportedDistributions;
769+
770+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
771+
772+
Assert.IsTrue(component.IsSupported());
773+
}
774+
775+
[Test]
776+
[TestCase("Debian,Ubuntu")]
777+
[TestCase("Ubuntu")]
778+
[TestCase("Fedora")]
779+
public void VirtualClientComponentIsNotSupportedWhenTheLinuxDistributionDoesNotMatchTheSupportedLinuxDistributions(string supportedDistributions)
780+
{
781+
this.mockFixture.Setup(PlatformID.Unix);
782+
this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora);
783+
this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = supportedDistributions;
784+
785+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
786+
787+
Assert.IsFalse(component.IsSupported());
788+
}
789+
790+
[Test]
791+
public void VirtualClientComponentIsSupportedWhenTheLinuxDistributionMatchesADownstreamDistribution()
792+
{
793+
// Ubuntu is a downstream distribution of Debian. The match is made on the distribution
794+
// itself and not on the upstream distribution.
795+
this.mockFixture.Setup(PlatformID.Unix);
796+
this.SetupLinuxDistribution(LinuxDistribution.Ubuntu, LinuxUpstreamDistribution.Debian);
797+
this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "Debian,Ubuntu";
798+
799+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
800+
801+
Assert.IsTrue(component.IsSupported());
802+
}
803+
804+
[Test]
805+
public void VirtualClientComponentIsNotSupportedOnNonLinuxSystemsWhenTheSupportedLinuxDistributionsParameterIsDefined()
806+
{
807+
// A component defining the parameter is describing Linux-specific behavior and thus
808+
// is never supported on non-Linux systems.
809+
this.mockFixture.Setup(PlatformID.Win32NT);
810+
this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "Ubuntu";
811+
812+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
813+
814+
Assert.IsFalse(component.IsSupported());
815+
}
816+
817+
[Test]
818+
public void VirtualClientComponentIsNotSupportedWhenTheSupportedPlatformsDoNotMatchEvenIfTheLinuxDistributionMatches()
819+
{
820+
this.mockFixture.Setup(PlatformID.Unix, System.Runtime.InteropServices.Architecture.X64);
821+
this.SetupLinuxDistribution(LinuxDistribution.AzureLinux, LinuxUpstreamDistribution.Fedora);
822+
this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedPlatforms)] = "linux-arm64";
823+
this.mockFixture.Parameters[nameof(VirtualClientComponent.SupportedLinuxDistributions)] = "AzureLinux";
824+
825+
TestVirtualClientComponent component = new TestVirtualClientComponent(this.mockFixture.Dependencies, this.mockFixture.Parameters);
826+
827+
Assert.IsFalse(component.IsSupported());
828+
}
829+
830+
private void SetupLinuxDistribution(LinuxDistribution distribution, LinuxUpstreamDistribution upstreamDistribution)
831+
{
832+
this.mockFixture.SystemManagement
833+
.Setup(sm => sm.GetLinuxDistributionAsync(It.IsAny<CancellationToken>()))
834+
.ReturnsAsync(new LinuxDistributionInfo
835+
{
836+
Name = distribution.ToString(),
837+
Distribution = distribution,
838+
UpstreamDistribution = upstreamDistribution
839+
});
840+
}
841+
727842
private class TestVirtualClientComponent : VirtualClientComponent
728843
{
729844
public TestVirtualClientComponent(VirtualClientComponent component)
@@ -743,6 +858,11 @@ public TestVirtualClientComponent(IServiceCollection dependencies, IDictionary<s
743858
return base.IsInRole(role);
744859
}
745860

861+
public new bool IsSupported()
862+
{
863+
return base.IsSupported();
864+
}
865+
746866
protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken)
747867
{
748868
this.OnExecute?.Invoke(telemetryContext, cancellationToken);

‎src/VirtualClient/VirtualClient.Contracts/VirtualClientComponent.cs‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -633,6 +633,19 @@ protected set
633633
/// </summary>
634634
public DateTime StartTime { get; private set; }
635635

636+
/// <summary>
637+
/// Parameter describes the Linux distributions (e.g. AzureLinux, Ubuntu) for which the component
638+
/// is supported. A component defining this parameter is never executed on non-Linux systems.
639+
/// </summary>
640+
public IEnumerable<string> SupportedLinuxDistributions
641+
{
642+
get
643+
{
644+
this.Parameters.TryGetCollection<string>(nameof(this.SupportedLinuxDistributions), out IEnumerable<string> distributions);
645+
return distributions ?? Array.Empty<string>();
646+
}
647+
}
648+
636649
/// <summary>
637650
/// Parameter describes the platform/architectures for which the component is supported.
638651
/// </summary>
@@ -943,6 +956,10 @@ protected virtual bool IsSupported()
943956
{
944957
isSupported = false;
945958
}
959+
else if (this.SupportedLinuxDistributions?.Any() == true && !this.IsSupportedLinuxDistribution())
960+
{
961+
isSupported = false;
962+
}
946963
else if (this.Layout?.Clients?.Count() >= 2 && this.Roles?.Any() == true)
947964
{
948965
// Execution Criteria
@@ -1018,5 +1035,21 @@ private bool IsMe(ClientInstance clientInstance)
10181035

10191036
return isMatch;
10201037
}
1038+
1039+
private bool IsSupportedLinuxDistribution()
1040+
{
1041+
bool isSupported = false;
1042+
if (this.Platform == PlatformID.Unix)
1043+
{
1044+
LinuxDistributionInfo distribution = this.systemInfo.GetLinuxDistributionAsync(CancellationToken.None)
1045+
.GetAwaiter().GetResult();
1046+
1047+
isSupported = this.SupportedLinuxDistributions.Contains(
1048+
distribution.Distribution.ToString(),
1049+
StringComparer.OrdinalIgnoreCase);
1050+
}
1051+
1052+
return isSupported;
1053+
}
10211054
}
10221055
}

0 commit comments

Comments
 (0)