diff --git a/VERSION b/VERSION index 4772543317..619b537668 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -3.3.2 +3.3.3 diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyClientExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyClientExecutorTests.cs new file mode 100644 index 0000000000..2b0a91b0e5 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyClientExecutorTests.cs @@ -0,0 +1,423 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Net.Http; + using System.Reflection; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Actions; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using Newtonsoft.Json.Linq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class ElasticsearchRallyClientExecutorTests : MockFixture + { + private IEnumerable disks; + [SetUp] + public void SetupTest() + { + this.Setup(PlatformID.Unix); + + this.File.Reset(); + this.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.FileSystem.SetupGet(fs => fs.File).Returns(this.File.Object); + + string agentId = $"{Environment.MachineName}"; + this.SystemManagement.SetupGet(obj => obj.AgentId).Returns(agentId); + + this.disks = this.CreateDisks(PlatformID.Unix, true); + + this.DiskManager.Setup(mgr => mgr.GetDisksAsync(It.IsAny())).ReturnsAsync(() => this.disks); + + this.ApiClient.Setup(client => client.GetHeartbeatAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.ApiClient.Setup(client => client.GetServerOnlineStatusAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.Parameters = new Dictionary() + { + { nameof(ElasticsearchRallyClientExecutor.DiskFilter), "osdisk:false&biggestsize" }, + { nameof(ElasticsearchRallyClientExecutor.ElasticsearchVersion), "9.2.3" }, + { nameof(ElasticsearchRallyClientExecutor.RallyVersion), "2.12.0" }, + { nameof(ElasticsearchRallyClientExecutor.Port), "9200" }, + { nameof(ElasticsearchRallyClientExecutor.Scenario), "ExecuteGeoNamesBenchmark" }, + { nameof(ElasticsearchRallyClientExecutor.RallyTrackName), "geonames" }, + }; + } + + [Test] + [TestCase(false, false)] + [TestCase(false, true)] + [TestCase(true, true)] + public void TestElasticsearchRallyClientExecutorWhenReportNotGenerated(bool rallyConfigured, bool serverAvailable) + { + SetupTest(); + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = rallyConfigured, + })); + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.ServerAvailable = serverAvailable; + Assert.ThrowsAsync(() => executor.ExecuteAsync(EventContext.None, CancellationToken.None)); + } + } + + [Test] + public void TestElasticsearchRallyClientExecutorWhenRallyConfiguredAndReportFailure() + { + SetupTest(); + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = true, + })); + + bool commandExecuted = false; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + if (command == "/usr/bin/sudo" && arguments.Contains("python3 -m pipx run esrally race")) + { + commandExecuted = true; + } + }; + + executor.ReportCsvExists = true; + + Assert.ThrowsAsync(() => executor.ExecuteAsync(EventContext.None, CancellationToken.None)); + } + + Assert.IsTrue(commandExecuted); + } + + [Test] + public void TestElasticsearchRallyClientExecutorWhenRallyConfiguredAndReportWithInsufficientData() + { + SetupTest(); + + bool logMessageCaptured = false; + + // Use VirtualClient's LogMessage extension method + this.Logger.OnLog = (level, eventId, state, exception) => + { + if (eventId.Name.Contains("RallyReportCsvInsufficientData")) + { + logMessageCaptured = true; + } + }; + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = true, + })); + + bool commandExecuted = false; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + if (command == "/usr/bin/sudo" && arguments.Contains("python3 -m pipx run esrally race")) + { + commandExecuted = true; + } + }; + + executor.ReportCsvExists = true; + executor.ReportLines = new string[] + { + "Metric,Task,Value,Unit" + }; + + executor.ExecuteAsync(EventContext.None, CancellationToken.None); + } + + Assert.IsTrue(logMessageCaptured); + Assert.IsTrue(commandExecuted); + } + + [Test] + public void TestElasticsearchRallyClientExecutorWhenRallyMetricReaderFailure() + { + SetupTest(); + + this.Parameters.Remove(nameof(ElasticsearchRallyClientExecutor.Scenario)); + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = true, + })); + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.ReportCsvExists = true; + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + executor.ReportLines = System.IO.File.ReadAllLines(Path.Combine(currentDirectory, "Examples", "ElasticsearchRally", "ElasticsearchRallyExample.txt")); + + Assert.ThrowsAsync(() => executor.ExecuteAsync(EventContext.None, CancellationToken.None)); + + } + } + [Test] + public async Task TestElasticsearchRallyClientExecutorWhenRallyConfiguredAndReportGenerated() + { + SetupTest(); + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = true, + })); + + bool commandExecuted = false; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + if (command == "/usr/bin/sudo" && arguments.Contains("python3 -m pipx run esrally race")) + { + commandExecuted = true; + } + }; + + executor.ReportCsvExists = true; + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + executor.ReportLines = System.IO.File.ReadAllLines(Path.Combine(currentDirectory, "Examples", "ElasticsearchRally", "ElasticsearchRallyExample.txt")); + + await executor.ExecuteAsync(EventContext.None, CancellationToken.None); + } + + Assert.IsTrue(commandExecuted); + } + + [Test] + public void RallyChallengePropertyReturnsExpectedValue() + { + SetupTest(); + + this.Parameters[nameof(ElasticsearchRallyClientExecutor.RallyCommandLineArguments)] = "--challenge=append-no-conflicts"; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + Assert.AreEqual("--challenge=append-no-conflicts", executor.RallyCommandLineArguments); + } + } + + [Test] + public void RallyChallengePropertyReturnsEmptyStringWhenNotSpecified() + { + SetupTest(); + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + Assert.AreEqual(string.Empty, executor.RallyCommandLineArguments); + } + } + + [Test] + public async Task RallyChallengeIsIncludedInRallyCommandWhenSpecified() + { + SetupTest(); + + this.Parameters[nameof(ElasticsearchRallyClientExecutor.RallyCommandLineArguments)] = "--challenge=append-no-conflicts"; + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = true, + })); + + bool challengeIncludedInCommand = false; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + if (command == "/usr/bin/sudo" && arguments.Contains("python3 -m pipx run esrally race") && arguments.Contains("--challenge=append-no-conflicts")) + { + challengeIncludedInCommand = true; + } + }; + + executor.ReportCsvExists = true; + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + executor.ReportLines = System.IO.File.ReadAllLines(Path.Combine(currentDirectory, "Examples", "ElasticsearchRally", "ElasticsearchRallyExample.txt")); + + await executor.ExecuteAsync(EventContext.None, CancellationToken.None); + } + + Assert.IsTrue(challengeIncludedInCommand); + } + + [Test] + public void RallyIncludeTasksPropertyReturnsExpectedValue() + { + SetupTest(); + + this.Parameters[nameof(ElasticsearchRallyClientExecutor.RallyCommandLineArguments)] = "--include-tasks=index,search"; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + Assert.AreEqual("--include-tasks=index,search", executor.RallyCommandLineArguments); + } + } + + [Test] + public async Task RallyIncludeTasksIsIncludedInRallyCommandWhenSpecified() + { + SetupTest(); + + this.Parameters[nameof(ElasticsearchRallyClientExecutor.RallyCommandLineArguments)] = "--include-tasks=index,search"; + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = true, + })); + + bool includeTasksInCommand = false; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + if (command == "/usr/bin/sudo" && arguments.Contains("python3 -m pipx run esrally race") && arguments.Contains("--include-tasks=index,search")) + { + includeTasksInCommand = true; + } + }; + + executor.ReportCsvExists = true; + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + executor.ReportLines = System.IO.File.ReadAllLines(Path.Combine(currentDirectory, "Examples", "ElasticsearchRally", "ElasticsearchRallyExample.txt")); + + await executor.ExecuteAsync(EventContext.None, CancellationToken.None); + } + + Assert.IsTrue(includeTasksInCommand); + } + + [Test] + public void RallyCommandLineArgumentsPropertyReturnsExpectedValue() + { + SetupTest(); + + string customCommand = "race --track=geonames --target-hosts=localhost:9200 --custom-flag"; + this.Parameters[nameof(ElasticsearchRallyClientExecutor.RallyCommandLineArguments)] = customCommand; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + Assert.AreEqual(customCommand, executor.RallyCommandLineArguments); + } + } + + [Test] + public void RallyCommandLineArgumentsPropertyReturnsEmptyStringWhenNotSpecified() + { + SetupTest(); + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + Assert.AreEqual(string.Empty, executor.RallyCommandLineArguments); + } + } + + [Test] + public async Task RallyCommandLineArgumentsOverridesDefaultCommandWhenSpecified() + { + SetupTest(); + + string customCommand = "--custom-flag"; + this.Parameters[nameof(ElasticsearchRallyClientExecutor.RallyTrackName)] = "custom"; + this.Parameters[nameof(ElasticsearchRallyClientExecutor.RallyCommandLineArguments)] = customCommand; + + this.StateManager.OnGetState().ReturnsAsync(JObject.FromObject(new ElasticsearchRallyState() + { + RallyConfigured = true, + })); + + bool customCommandUsed = false; + + using (TestElasticsearchRallyClientExecutor executor = new TestElasticsearchRallyClientExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + if (command == "/usr/bin/sudo" && arguments.Contains("python3 -m pipx run esrally race --track=custom") && arguments.Contains("--custom-flag")) + { + customCommandUsed = true; + } + }; + + executor.ReportCsvExists = true; + string currentDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + executor.ReportLines = System.IO.File.ReadAllLines(Path.Combine(currentDirectory, "Examples", "ElasticsearchRally", "ElasticsearchRallyExample.txt")); + + await executor.ExecuteAsync(EventContext.None, CancellationToken.None); + } + + Assert.IsTrue(customCommandUsed); + } + + private class TestElasticsearchRallyClientExecutor : ElasticsearchRallyClientExecutor + { + public Action OnRunCommand { get; set; } + public bool ServerAvailable { get; set; } + public bool ReportCsvExists { get; set; } + public string[] ReportLines { get; set; } + + public TestElasticsearchRallyClientExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + } + + public new Task ExecuteAsync(EventContext context, CancellationToken cancellationToken) + { + return base.ExecuteAsync(context, cancellationToken); + } + + protected override bool RunCommand(EventContext telemetryContext, CancellationToken cancellationToken, string command, string arguments, out string output, out string error) + { + output = string.Empty; + error = string.Empty; + + OnRunCommand?.Invoke(command, arguments); + + return true; + } + + protected override bool CheckServerAvailable(EventContext telemetryContext, CancellationToken cancellationToken, string targetHost, int port, int timeout) + { + return this.ServerAvailable; + } + + protected override bool CheckFileExists(string path) + { + return this.ReportCsvExists; + } + + override protected string ReadReportFile(string reportPath) + { + return string.Join(Environment.NewLine, this.ReportLines); + } + } + + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyMetricsParserTests.cs new file mode 100644 index 0000000000..87dc85d8c1 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyMetricsParserTests.cs @@ -0,0 +1,884 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Reflection; + using VirtualClient; + using VirtualClient.Contracts; + using NUnit.Framework; + + [TestFixture] + [Category("Unit")] + public class ElasticsearchRallyMetricsParserTests + { + private string examplePath; + + [SetUp] + public void Setup() + { + string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); + this.examplePath = Path.Combine(workingDirectory, "Examples", "ElasticsearchRally"); + } + + [Test] + public void ParserThrowsArgumentNullExceptionWhenReportContentsIsNull() + { + Assert.Throws(() => new ElasticsearchRallyMetricsParser( + null, + new Dictionary(), + false)); + } + + [Test] + public void ParserThrowsArgumentNullExceptionWhenReportContentsIsEmpty() + { + Assert.Throws(() => new ElasticsearchRallyMetricsParser( + string.Empty, + new Dictionary(), + false)); + } + + [Test] + public void ParserThrowsArgumentNullExceptionWhenMetadataIsNull() + { + Assert.Throws(() => new ElasticsearchRallyMetricsParser( + "Metric,Task,Value,Unit", + null, + false)); + } + + [Test] + public void ParserHandlesInvalidLinesGracefully() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "Invalid line without enough columns", + "Median throughput,index,45000.5,docs/s", + "Invalid,metric,value,ms", + "50th percentile latency,search,notanumber,ms" + }); + + Dictionary metadata = new Dictionary + { + { "track", "test" } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + // Only one valid metric line exists (with parseable value) + Assert.AreEqual(1, metrics.Count, $"Expected 1 metric but got {metrics.Count}"); + + // Verify the one valid metric was parsed correctly + // "Median throughput" -> "throughput Median" -> "index throughput Median" -> "index-throughput-Median" + Assert.AreEqual("index-throughput-Median", metrics[0].Name); + Assert.AreEqual(45000.5, metrics[0].Value); + Assert.AreEqual("docs/s", metrics[0].Unit); + Assert.AreEqual(MetricRelativity.HigherIsBetter, metrics[0].Relativity); + } + + [Test] + public void ParserParsesBasicMetricsCorrectly() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "Median throughput,index,45000.5,docs/s", + "50th percentile latency,search,150.25,ms" + }); + + Dictionary metadata = new Dictionary + { + { "track", "geonames" }, + { "challenge", "append-no-conflicts" } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.IsNotNull(metrics); + Assert.AreEqual(2, metrics.Count); + + // "Median throughput" is transformed to "throughput Median" (NOT "throughput P50") + Metric throughputMetric = metrics.First(m => m.Name == "index-throughput-Median"); + Assert.AreEqual(45000.5, throughputMetric.Value); + Assert.AreEqual("docs/s", throughputMetric.Unit); + Assert.AreEqual(MetricRelativity.HigherIsBetter, throughputMetric.Relativity); + Assert.AreEqual(1, throughputMetric.Verbosity); // Contains "throughput" -> Verbosity 1 + + // "50th percentile latency" is transformed to "latency P50" + Metric latencyMetric = metrics.First(m => m.Name == "search-latency-P50"); + Assert.AreEqual(150.25, latencyMetric.Value); + Assert.AreEqual("ms", latencyMetric.Unit); + Assert.AreEqual(MetricRelativity.LowerIsBetter, latencyMetric.Relativity); + Assert.AreEqual(1, latencyMetric.Verbosity); // Contains "latency" -> Verbosity 1 + } + + [Test] + public void ParserParsesExampleFileCorrectly() + { + string exampleFile = Path.Combine(this.examplePath, "ElasticsearchRallyExample.txt"); + string reportContents = File.ReadAllText(exampleFile); + + Dictionary metadata = new Dictionary + { + { "track", "geonames" } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + Assert.IsNotNull(metrics); + Assert.IsTrue(metrics.Count > 0); + } + + [Test] + public void ParserTransformsMetricNamesCorrectly() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,task1,100,docs/s", + "mean throughput,task2,200,docs/s", + "100th percentile latency,task3,300,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(3, metrics.Count); + + // "median throughput" -> "throughput Median" (NOT "throughput P50") + Metric metric1 = metrics.First(m => m.Name == "task1-throughput-Median"); + Assert.IsNotNull(metric1); + Assert.AreEqual(100, metric1.Value); + Assert.AreEqual(1, metric1.Verbosity); // Contains "throughput" and ends with "Median" -> Verbosity 1 + + // "mean throughput" -> "throughput Mean" + Metric metric2 = metrics.First(m => m.Name == "task2-throughput-Mean"); + Assert.IsNotNull(metric2); + Assert.AreEqual(200, metric2.Value); + Assert.AreEqual(1, metric2.Verbosity); // Contains "throughput" and ends with "Mean" -> Verbosity 1 + + // "100th percentile latency" -> "latency P100" + Metric metric3 = metrics.First(m => m.Name == "task3-latency-P100"); + Assert.IsNotNull(metric3); + Assert.AreEqual(300, metric3.Value); + Assert.AreEqual(1, metric3.Verbosity); // Contains "latency" and ends with "P100" -> Verbosity 1 + } + + [Test] + public void ParserSetsCorrectRelativityForThroughputMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "Median throughput,index,45000.5,docs/s", + "Mean throughput,search,30000,docs/s" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.IsTrue(metrics.All(m => m.Relativity == MetricRelativity.HigherIsBetter)); + } + + [Test] + public void ParserSetsCorrectRelativityForLatencyMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "50th percentile latency,search,150.25,ms", + "90th percentile latency,search,200.5,ms", + "99th percentile latency,search,350.75,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.IsTrue(metrics.All(m => m.Relativity == MetricRelativity.LowerIsBetter)); + } + + [Test] + public void ParserSetsCorrectRelativityForServiceTimeMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "50th percentile service time,search,120.5,ms", + "100th percentile service time,index,250.75,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.IsTrue(metrics.All(m => m.Relativity == MetricRelativity.LowerIsBetter)); + } + + [Test] + public void ParserSetsCorrectRelativityForRateMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "error rate,search,0.5,%" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(MetricRelativity.LowerIsBetter, metrics[0].Relativity); + } + + [Test] + public void ParserFiltersRelevantMetricsWhenCollectAllMetricsIsFalse() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "Median throughput,index,1000,docs/s", + "50th percentile latency,search,100,ms", + "Some other metric,task,50,units" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + // Should only collect relevant metrics + Assert.IsTrue(metrics.Count <= 2); + Assert.IsTrue(metrics.All(m => m.Name.Contains("throughput") || m.Name.Contains("latency"))); + } + + [Test] + public void ParserCollectsAllMetricsWhenCollectAllMetricsIsTrue() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "Median throughput,index,1000,docs/s", + "50th percentile latency,search,100,ms", + "Some other metric,task,50,units" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(3, metrics.Count); + } + + [Test] + public void ParserHandlesMetricsWithoutTaskNames() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median cumulative indexing time across primary shards,,1500,ms", + "dataset size,,10240,MB" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + Assert.IsTrue(metrics.Count > 0); + // Metrics without task names should not have task prefix + Assert.IsTrue(metrics.All(m => !m.Name.StartsWith("-"))); + } + + [Test] + public void ParserHandlesCountMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "segment count,,100," + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(1, metrics.Count); + Assert.AreEqual("count", metrics[0].Unit); + } + + [Test] + public void ParserIncludesMetadataInParsedMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "Median throughput,index,45000.5,docs/s" + }); + + Dictionary metadata = new Dictionary + { + { "track", "geonames" }, + { "challenge", "append-no-conflicts" }, + { "version", "8.0.0" } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(1, metrics.Count); + Assert.IsTrue(metrics[0].Metadata.ContainsKey("track")); + Assert.AreEqual("geonames", metrics[0].Metadata["track"]); + Assert.IsTrue(metrics[0].Metadata.ContainsKey("challenge")); + Assert.AreEqual("append-no-conflicts", metrics[0].Metadata["challenge"]); + Assert.IsTrue(metrics[0].Metadata.ContainsKey("version")); + Assert.AreEqual("8.0.0", metrics[0].Metadata["version"]); + } + + [Test] + public void ParserReplacesSpacesWithHyphensInMetricNames() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "50th percentile latency,search query,150.25,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(1, metrics.Count); + Assert.IsFalse(metrics[0].Name.Contains(" ")); + Assert.IsTrue(metrics[0].Name.Contains("-")); + Assert.AreEqual("search-query-latency-P50", metrics[0].Name); + } + + [Test] + public void ParserHandlesTimeUnitsCorrectly() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "some metric,task1,100,s", + "another metric,task2,200,ms", + "third metric,task3,300,min" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + // Time units should have LowerIsBetter relativity if not otherwise specified + Assert.AreEqual(3, metrics.Count); + Assert.IsTrue(metrics.All(m => m.Relativity == MetricRelativity.LowerIsBetter)); + } + + [Test] + public void ParserHandlesCaseInsensitiveMetricNames() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "MEDIAN THROUGHPUT,index,1000,docs/s", + "50TH PERCENTILE LATENCY,search,100,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(2, metrics.Count); + Assert.IsTrue(metrics.Any(m => m.Name.Contains("throughput"))); + Assert.IsTrue(metrics.Any(m => m.Name.Contains("latency"))); + } + + #region MetricsVerbosity Tests + + [Test] + public void MetricsVerbosityPropertyReturnsDefaultValueWhenNotSpecified() + { + string reportContents = "Metric,Task,Value,Unit\nMedian throughput,index,1000,docs/s"; + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + Assert.AreEqual(1, parser.MetricsVerbosity); + } + + [Test] + public void MetricsVerbosityPropertyReturnsSpecifiedValue() + { + string reportContents = "Metric,Task,Value,Unit\nMedian throughput,index,1000,docs/s"; + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", 3 } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + Assert.AreEqual(3, parser.MetricsVerbosity); + } + + [Test] + public void MetricsVerbosityPropertyReturnsDefaultValueWhenInvalidValue() + { + string reportContents = "Metric,Task,Value,Unit\nMedian throughput,index,1000,docs/s"; + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", "invalid" } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + Assert.AreEqual(1, parser.MetricsVerbosity); + } + + [Test] + public void ParserFiltersMetricsBasedOnVerbosityLevel1() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,index,1000,docs/s", // Verbosity 1 (contains throughput) + "100th percentile latency,search,100,ms", // Verbosity 1 (P100 + contains latency) + "50th percentile latency,query,50,ms", // Verbosity 2 (P50) but contains latency -> Verbosity 1 + "90th percentile latency,query,90,ms", // Verbosity 2 (P90) but contains latency -> Verbosity 1 + "mean throughput,index,500,docs/s", // Verbosity 1 (Mean + contains throughput) + "some custom metric,task,123,units" // Verbosity 5 + }); + + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", 1 } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + // Should include all metrics containing latency or throughput (Verbosity 1) + Assert.AreEqual(5, metrics.Count); + Assert.IsTrue(metrics.All(m => m.Verbosity == 1)); + } + + [Test] + public void ParserFiltersMetricsBasedOnVerbosityLevel2() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,index,1000,docs/s", // Verbosity 1 + "100th percentile latency,search,100,ms", // Verbosity 1 + "50th percentile latency,query,50,ms", // Verbosity 1 (contains latency) + "90th percentile latency,query,90,ms", // Verbosity 1 (contains latency) + "99th percentile latency,query,99,ms", // Verbosity 1 (contains latency) + "mean throughput,index,500,docs/s", // Verbosity 1 + "some custom metric,task,123,units" // Verbosity 5 + }); + + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", 2 } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + // Should include Verbosity 1 and 2 metrics (all except the custom metric) + Assert.AreEqual(6, metrics.Count); + Assert.IsTrue(metrics.All(m => m.Verbosity <= 2)); + Assert.IsFalse(metrics.Any(m => m.Name.Contains("custom"))); + } + + [Test] + public void ParserFiltersMetricsBasedOnVerbosityLevel5() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,index,1000,docs/s", // Verbosity 1 + "50th percentile latency,query,50,ms", // Verbosity 1 (contains latency) + "mean throughput,index,500,docs/s", // Verbosity 1 + "dataset size,,10240,MB" // Verbosity 5 (summary metric, no task) + }); + + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", 5 } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + // Should include Verbosity 1 and Verbosity 5 metrics when MetricsVerbosity is set to 5. + Assert.AreEqual(4, metrics.Count); + Assert.AreEqual(3, metrics.Count(m => m.Verbosity == 1)); + Assert.AreEqual(1, metrics.Count(m => m.Verbosity == 5)); + Assert.IsTrue(metrics.Any(m => m.Name.Contains("dataset") && m.Verbosity == 5)); + } + + [Test] + public void ParserIgnoresVerbosityFilteringWhenCollectAllMetricsIsTrue() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,index,1000,docs/s", // Verbosity 1 + "50th percentile latency,query,50,ms", // Verbosity 1 + "mean throughput,index,500,docs/s", // Verbosity 1 + "some custom metric,task,123,units" // Verbosity 5 + }); + + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", 1 } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + // Should include all metrics regardless of verbosity when collectAllMetrics is true + Assert.AreEqual(4, metrics.Count); + } + + [Test] + public void ParserAssignsCorrectVerbosityToMedianMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,index,1000,docs/s", + "median latency,search,50,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(2, metrics.Count); + Assert.IsTrue(metrics.All(m => m.Verbosity == 1)); + } + + [Test] + public void ParserAssignsCorrectVerbosityToP100Metrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "100th percentile latency,search,200,ms", + "100th percentile service time,index,150,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(2, metrics.Count); + // Both contain "latency" or have P100 ending, so Verbosity 1 + Assert.IsTrue(metrics.All(m => m.Verbosity == 1)); + } + + [Test] + public void ParserAssignsCorrectVerbosityToP50P90P99Metrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "50th percentile service time,search,50,ms", + "90th percentile service time,search,90,ms", + "99th percentile service time,search,99,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(3, metrics.Count); + // P50, P90, P99 without latency/throughput -> Verbosity 2 + Assert.IsTrue(metrics.All(m => m.Verbosity == 2)); + } + + [Test] + public void ParserAssignsVerbosity1ToLatencyMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "50th percentile latency,search,50,ms", + "90th percentile latency,search,90,ms", + "99th percentile latency,search,99,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(3, metrics.Count); + // All contain "latency" -> Verbosity 1 + Assert.IsTrue(metrics.All(m => m.Verbosity == 1)); + } + + [Test] + public void ParserAssignsVerbosity1ToThroughputMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,index,1000,docs/s", + "mean throughput,index,500,docs/s" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(2, metrics.Count); + // Both contain "throughput" -> Verbosity 1 + Assert.IsTrue(metrics.All(m => m.Verbosity == 1)); + } + + [Test] + public void ParserAssignsVerbosity1ToMeanMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "mean throughput,index,500,docs/s", + "mean service time,search,100,ms" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(2, metrics.Count); + // Mean metrics with throughput/other -> Verbosity 1 + Assert.IsTrue(metrics.All(m => m.Verbosity == 1)); + } + + [Test] + public void ParserAssignsCorrectVerbosityToOtherMetrics() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "error rate,search,0.1,%", + "some custom metric,task,123,units" + }); + + Dictionary metadata = new Dictionary(); + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: true); + + IList metrics = parser.Parse(); + + Assert.AreEqual(2, metrics.Count); + // Neither contains latency/throughput, nor ends with P50/P90/P99/P100/Mean/Median -> Verbosity 5 + Assert.IsTrue(metrics.All(m => m.Verbosity == 5)); + } + + [Test] + public void ParserCombinesVerbosityFilteringWithRelevantMetricsFiltering() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median throughput,index,1000,docs/s", // Relevant, Verbosity 1 + "50th percentile latency,search,50,ms", // Relevant, Verbosity 1 (contains latency) + "mean throughput,index,500,docs/s", // Relevant, Verbosity 1 + "some irrelevant metric,task,100,units" // Not relevant + }); + + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", 2 } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + // Should include only relevant metrics with verbosity <= 2 + Assert.AreEqual(3, metrics.Count); + Assert.IsTrue(metrics.All(m => m.Verbosity <= 2)); + } + + [Test] + public void ParserHandlesSummaryMetricsWithVerbosityFiltering() + { + string reportContents = string.Join(Environment.NewLine, new[] + { + "Metric,Task,Value,Unit", + "median cumulative indexing time across primary shards,,1500,ms", + "median cumulative merge time across primary shards,,500,ms", + "total young gen gc time,,200,s", + "dataset size,,10240,MB" + }); + + Dictionary metadata = new Dictionary + { + { "MetricsVerbosity", 1 } + }; + + ElasticsearchRallyMetricsParser parser = new ElasticsearchRallyMetricsParser( + reportContents, + metadata, + rallyCollectAllMetrics: false); + + IList metrics = parser.Parse(); + + // Summary metrics with "median" -> Verbosity 1 + Assert.IsTrue(metrics.Count >= 2); + Assert.IsTrue(metrics.Any(m => m.Name.Contains("indexing") && m.Verbosity == 1)); + Assert.IsTrue(metrics.Any(m => m.Name.Contains("merge") && m.Verbosity == 1)); + } + + #endregion + } +} \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyServerExecutorTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyServerExecutorTests.cs new file mode 100644 index 0000000000..96e587be49 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/ElasticsearchRally/ElasticsearchRallyServerExecutorTests.cs @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net.Http; + using System.Threading; + using System.Threading.Tasks; + using VirtualClient.Actions; + using Microsoft.CodeAnalysis; + using Microsoft.Extensions.DependencyInjection; + using Moq; + using NUnit.Framework; + using Polly; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + [TestFixture] + [Category("Unit")] + public class ElasticsearchRallyServerExecutorTests : MockFixture + { + private IEnumerable disks; + [SetUp] + public void SetupTest() + { + this.Setup(PlatformID.Unix); + + this.File.Reset(); + this.File.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.Directory.Setup(f => f.Exists(It.IsAny())) + .Returns(true); + this.FileSystem.SetupGet(fs => fs.File).Returns(this.File.Object); + + string agentId = $"{Environment.MachineName}"; + this.SystemManagement.SetupGet(obj => obj.AgentId).Returns(agentId); + + this.disks = this.CreateDisks(PlatformID.Unix, true); + + this.DiskManager.Setup(mgr => mgr.GetDisksAsync(It.IsAny())).ReturnsAsync(() => this.disks); + + this.ApiClient.Setup(client => client.GetHeartbeatAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.ApiClient.Setup(client => client.GetServerOnlineStatusAsync(It.IsAny(), It.IsAny>())) + .ReturnsAsync(this.CreateHttpResponse(System.Net.HttpStatusCode.OK)); + + this.Parameters = new Dictionary() + { + { nameof(ElasticsearchRallyServerExecutor.DiskFilter), "osdisk:false&biggestsize" }, + { nameof(ElasticsearchRallyServerExecutor.Port), "9200" }, + { nameof(ElasticsearchRallyServerExecutor.PackageName), "elasticsearchrally" }, + }; + } + + [Test] + public void TestElasticsearchRallyServerExecutorInitializeYmlNotFound() + { + SetupTest(); + + bool commandExecuted = false; + + using (TestElasticsearchRallyServerExecutor executor = new TestElasticsearchRallyServerExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + commandExecuted = true; + }; + + Assert.ThrowsAsync(() => executor.InitializeAsync(EventContext.None, CancellationToken.None)); + } + + Assert.IsTrue(commandExecuted); + } + + [Test] + [TestCase(true)] + [TestCase(false)] + public async Task TestElasticsearchRallyServerExecutorInitialize(bool useWget) + { + SetupTest(); + + bool commandExecuted = false; + + this.Parameters.Add("UseWgetForElasticsearhDownloadOnLinux", useWget); + + using (TestElasticsearchRallyServerExecutor executor = new TestElasticsearchRallyServerExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + commandExecuted = true; + }; + + executor.FileExists = true; + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + } + + Assert.IsTrue(commandExecuted); + } + + [Test] + public async Task TestElasticsearchRallyServerExecutorInitializeUseDefaultElasticsearchPath() + { + SetupTest(); + + bool commandExecuted = false; + + using (TestElasticsearchRallyServerExecutor executor = new TestElasticsearchRallyServerExecutor(this.Dependencies, this.Parameters)) + { + executor.OnRunCommand = (command, arguments) => + { + commandExecuted = true; + }; + + executor.FileExists = true; + await executor.InitializeAsync(EventContext.None, CancellationToken.None); + } + + Assert.IsTrue(commandExecuted); + } + [Test] + public async Task TestElasticsearchRallyServerExecutorExpectedRun() + { + SetupTest(); + + bool commandExecuted = false; + + using (TestElasticsearchRallyServerExecutor executor = new TestElasticsearchRallyServerExecutor(this.Dependencies, this.Parameters)) + { + await executor.ExecuteAsync(EventContext.None, CancellationToken.None); + commandExecuted = true; + } + + Assert.IsTrue(commandExecuted); + } + + private class TestElasticsearchRallyServerExecutor : ElasticsearchRallyServerExecutor + { + public Action OnRunCommand { get; set; } + public Action OnFileCopy { get; set; } + public Action OnCreateDirectory { get; set; } + public Func OnDownloadFile { get; set; } + public bool FileExists { get; set; } + public bool DirectoryExists { get; set; } + public string DataDirectory { get; set; } + public string TestPlatformArchitectureName { get; set; } + public string MountPoint { get; set; } + + public TestElasticsearchRallyServerExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.DataDirectory = "/data"; + this.PackageName = "elasticsearchrally"; + this.WaitForElasticsearchAvailabilityTimeout = 0; + this.MountPoint = "/mnt"; + } + + public new Task InitializeAsync(EventContext context, CancellationToken cancellationToken) + { + return base.InitializeAsync(context, cancellationToken); + } + + public new Task ExecuteAsync(EventContext context, CancellationToken cancellationToken) + { + return base.ExecuteAsync(context, cancellationToken); + } + + protected override Task GetDataDirectoryAsync(CancellationToken cancellationToken) + { + return Task.FromResult(this.DataDirectory); + } + + protected override bool RunCommand(EventContext telemetryContext, CancellationToken cancellationToken, string command, string arguments, out string output, out string error) + { + output = string.Empty; + error = string.Empty; + + OnRunCommand?.Invoke(command, arguments); + + return true; + } + + protected override bool CheckFileExists(string path) + { + return this.FileExists; + } + + protected override bool CheckDirectoryExists(string path) + { + return this.DirectoryExists; + } + + protected override void WriteAllText(string path, string content) + { + + } + + protected override string ReadAllText(string path) + { + return "sample text"; + } + + protected override void FileCopy(string sourcePath, string destinationPath, bool overwrite) + { + OnFileCopy?.Invoke(sourcePath, destinationPath, overwrite); + } + + protected override void CreateDirectory(string path) + { + OnCreateDirectory?.Invoke(path); + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ElasticSearch/elasticsearch-output.json b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ElasticSearch/elasticsearch-output.json new file mode 100644 index 0000000000..023ea53509 Binary files /dev/null and b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ElasticSearch/elasticsearch-output.json differ diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ElasticsearchRally/ElasticsearchRallyExample.txt b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ElasticsearchRally/ElasticsearchRallyExample.txt new file mode 100644 index 0000000000..e141d35f5d --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/Examples/ElasticsearchRally/ElasticsearchRallyExample.txt @@ -0,0 +1,458 @@ +Metric,Task,Value,Unit +Cumulative indexing time of primary shards,,0.011383333333333334,min +Min cumulative indexing time across primary shards,,0.0015833333333333333,min +Median cumulative indexing time across primary shards,,0.00175,min +Max cumulative indexing time across primary shards,,0.0031833333333333336,min +Cumulative indexing throttle time of primary shards,,0,min +Min cumulative indexing throttle time across primary shards,,0,min +Median cumulative indexing throttle time across primary shards,,0,min +Max cumulative indexing throttle time across primary shards,,0,min +Cumulative merge time of primary shards,,0,min +Cumulative merge count of primary shards,,0, +Min cumulative merge time across primary shards,,0,min +Median cumulative merge time across primary shards,,0,min +Max cumulative merge time across primary shards,,0,min +Cumulative merge throttle time of primary shards,,0,min +Min cumulative merge throttle time across primary shards,,0,min +Median cumulative merge throttle time across primary shards,,0,min +Max cumulative merge throttle time across primary shards,,0,min +Cumulative refresh time of primary shards,,0.004366666666666667,min +Cumulative refresh count of primary shards,,35, +Min cumulative refresh time across primary shards,,0.0005333333333333334,min +Median cumulative refresh time across primary shards,,0.0006166666666666666,min +Max cumulative refresh time across primary shards,,0.0018833333333333334,min +Cumulative flush time of primary shards,,0,min +Cumulative flush count of primary shards,,0, +Min cumulative flush time across primary shards,,0,min +Median cumulative flush time across primary shards,,0,min +Max cumulative flush time across primary shards,,0,min +Total Young Gen GC time,,0,s +Total Young Gen GC count,,0, +Total Old Gen GC time,,0,s +Total Old Gen GC count,,0, +Dataset size,,0.00031846389174461365,GB +Store size,,0.00031846389174461365,GB +Translog size,,2.561137080192566e-07,GB +Heap used for segments,,0,MB +Heap used for doc values,,0,MB +Heap used for terms,,0,MB +Heap used for norms,,0,MB +Heap used for points,,0,MB +Heap used for stored fields,,0,MB +Segment count,,5, +Total Ingest Pipeline count,,0, +Total Ingest Pipeline time,,0,s +Total Ingest Pipeline failed,,0, +Min Throughput,index-append,3136.69,docs/s +Mean Throughput,index-append,3136.69,docs/s +Median Throughput,index-append,3136.69,docs/s +Max Throughput,index-append,3136.69,docs/s +50th percentile latency,index-append,411.4830879999545,ms +100th percentile latency,index-append,493.50839100009125,ms +50th percentile service time,index-append,411.4830879999545,ms +100th percentile service time,index-append,493.50839100009125,ms +error rate,index-append,0.00,% +Min Throughput,index-stats,106.55,ops/s +Mean Throughput,index-stats,106.55,ops/s +Median Throughput,index-stats,106.55,ops/s +Max Throughput,index-stats,106.55,ops/s +100th percentile latency,index-stats,17.315318999976625,ms +100th percentile service time,index-stats,7.617430999971475,ms +error rate,index-stats,0.00,% +Min Throughput,node-stats,35.37,ops/s +Mean Throughput,node-stats,35.37,ops/s +Median Throughput,node-stats,35.37,ops/s +Max Throughput,node-stats,35.37,ops/s +100th percentile latency,node-stats,41.9795630000408,ms +100th percentile service time,node-stats,13.148037000064505,ms +error rate,node-stats,0.00,% +Min Throughput,default,10.86,ops/s +Mean Throughput,default,10.86,ops/s +Median Throughput,default,10.86,ops/s +Max Throughput,default,10.86,ops/s +100th percentile latency,default,105.78448999990542,ms +100th percentile service time,default,13.30666100000144,ms +error rate,default,0.00,% +Min Throughput,term,79.80,ops/s +Mean Throughput,term,79.80,ops/s +Median Throughput,term,79.80,ops/s +Max Throughput,term,79.80,ops/s +100th percentile latency,term,23.220744000013838,ms +100th percentile service time,term,10.408558999984052,ms +error rate,term,0.00,% +Min Throughput,phrase,57.76,ops/s +Mean Throughput,phrase,57.76,ops/s +Median Throughput,phrase,57.76,ops/s +Max Throughput,phrase,57.76,ops/s +100th percentile latency,phrase,24.33359899998777,ms +100th percentile service time,phrase,6.615323000005446,ms +error rate,phrase,0.00,% +Min Throughput,country_agg_uncached,16.73,ops/s +Mean Throughput,country_agg_uncached,16.73,ops/s +Median Throughput,country_agg_uncached,16.73,ops/s +Max Throughput,country_agg_uncached,16.73,ops/s +100th percentile latency,country_agg_uncached,72.52013200002239,ms +100th percentile service time,country_agg_uncached,12.335570000004736,ms +error rate,country_agg_uncached,0.00,% +Min Throughput,country_agg_cached,64.70,ops/s +Mean Throughput,country_agg_cached,64.70,ops/s +Median Throughput,country_agg_cached,64.70,ops/s +Max Throughput,country_agg_cached,64.70,ops/s +100th percentile latency,country_agg_cached,39.73352600007729,ms +100th percentile service time,country_agg_cached,23.965268999972977,ms +error rate,country_agg_cached,0.00,% +Min Throughput,scroll,14.43,pages/s +Mean Throughput,scroll,14.43,pages/s +Median Throughput,scroll,14.43,pages/s +Max Throughput,scroll,14.43,pages/s +100th percentile latency,scroll,185.59198600007676,ms +100th percentile service time,scroll,46.108054000001175,ms +error rate,scroll,0.00,% +Min Throughput,expression,9.45,ops/s +Mean Throughput,expression,9.45,ops/s +Median Throughput,expression,9.45,ops/s +Max Throughput,expression,9.45,ops/s +100th percentile latency,expression,115.3483429999369,ms +100th percentile service time,expression,9.098823999920569,ms +error rate,expression,0.00,% +Min Throughput,painless_static,9.91,ops/s +Mean Throughput,painless_static,9.91,ops/s +Median Throughput,painless_static,9.91,ops/s +Max Throughput,painless_static,9.91,ops/s +100th percentile latency,painless_static,113.48900499990577,ms +100th percentile service time,painless_static,12.192466999977114,ms +error rate,painless_static,0.00,% +Min Throughput,painless_dynamic,19.61,ops/s +Mean Throughput,painless_dynamic,19.61,ops/s +Median Throughput,painless_dynamic,19.61,ops/s +Max Throughput,painless_dynamic,19.61,ops/s +100th percentile latency,painless_dynamic,64.49730999997882,ms +100th percentile service time,painless_dynamic,12.27088600001025,ms +error rate,painless_dynamic,0.00,% +Min Throughput,decay_geo_gauss_function_score,41.41,ops/s +Mean Throughput,decay_geo_gauss_function_score,41.41,ops/s +Median Throughput,decay_geo_gauss_function_score,41.41,ops/s +Max Throughput,decay_geo_gauss_function_score,41.41,ops/s +100th percentile latency,decay_geo_gauss_function_score,36.979236999968634,ms +100th percentile service time,decay_geo_gauss_function_score,12.484946999961721,ms +error rate,decay_geo_gauss_function_score,0.00,% +Min Throughput,decay_geo_gauss_script_score,33.34,ops/s +Mean Throughput,decay_geo_gauss_script_score,33.34,ops/s +Median Throughput,decay_geo_gauss_script_score,33.34,ops/s +Max Throughput,decay_geo_gauss_script_score,33.34,ops/s +100th percentile latency,decay_geo_gauss_script_score,42.514687999982925,ms +100th percentile service time,decay_geo_gauss_script_score,12.168380000048273,ms +error rate,decay_geo_gauss_script_score,0.00,% +Min Throughput,field_value_function_score,57.88,ops/s +Mean Throughput,field_value_function_score,57.88,ops/s +Median Throughput,field_value_function_score,57.88,ops/s +Max Throughput,field_value_function_score,57.88,ops/s +100th percentile latency,field_value_function_score,29.312013999970077,ms +100th percentile service time,field_value_function_score,11.531850999972448,ms +error rate,field_value_function_score,0.00,% +Min Throughput,field_value_script_score,38.28,ops/s +Mean Throughput,field_value_script_score,38.28,ops/s +Median Throughput,field_value_script_score,38.28,ops/s +Max Throughput,field_value_script_score,38.28,ops/s +100th percentile latency,field_value_script_score,36.996438000073795,ms +100th percentile service time,field_value_script_score,10.143301000084648,ms +error rate,field_value_script_score,0.00,% +Min Throughput,large_terms,1.70,ops/s +Mean Throughput,large_terms,1.70,ops/s +Median Throughput,large_terms,1.70,ops/s +Max Throughput,large_terms,1.70,ops/s +100th percentile latency,large_terms,839.6622349999916,ms +100th percentile service time,large_terms,244.41395700000612,ms +error rate,large_terms,0.00,% +Min Throughput,large_filtered_terms,8.92,ops/s +Mean Throughput,large_filtered_terms,8.92,ops/s +Median Throughput,large_filtered_terms,8.92,ops/s +Max Throughput,large_filtered_terms,8.92,ops/s +100th percentile latency,large_filtered_terms,217.69280099999833,ms +100th percentile service time,large_filtered_terms,98.75722699996459,ms +error rate,large_filtered_terms,0.00,% +Min Throughput,large_prohibited_terms,6.94,ops/s +Mean Throughput,large_prohibited_terms,6.94,ops/s +Median Throughput,large_prohibited_terms,6.94,ops/s +Max Throughput,large_prohibited_terms,6.94,ops/s +100th percentile latency,large_prohibited_terms,250.7793090001087,ms +100th percentile service time,large_prohibited_terms,100.76048100006574,ms +error rate,large_prohibited_terms,0.00,% +Min Throughput,desc_sort_population,30.44,ops/s +Mean Throughput,desc_sort_population,30.44,ops/s +Median Throughput,desc_sort_population,30.44,ops/s +Max Throughput,desc_sort_population,30.44,ops/s +100th percentile latency,desc_sort_population,45.147679000024254,ms +100th percentile service time,desc_sort_population,11.996705999990809,ms +error rate,desc_sort_population,0.00,% +Min Throughput,asc_sort_population,90.59,ops/s +Mean Throughput,asc_sort_population,90.59,ops/s +Median Throughput,asc_sort_population,90.59,ops/s +Max Throughput,asc_sort_population,90.59,ops/s +100th percentile latency,asc_sort_population,20.246841000016502,ms +100th percentile service time,asc_sort_population,8.775173999993058,ms +error rate,asc_sort_population,0.00,% +Min Throughput,asc_sort_with_after_population,73.41,ops/s +Mean Throughput,asc_sort_with_after_population,73.41,ops/s +Median Throughput,asc_sort_with_after_population,73.41,ops/s +Max Throughput,asc_sort_with_after_population,73.41,ops/s +100th percentile latency,asc_sort_with_after_population,19.789750999962052,ms +100th percentile service time,asc_sort_with_after_population,5.837562000010621,ms +error rate,asc_sort_with_after_population,0.00,% +Min Throughput,desc_sort_geonameid,60.90,ops/s +Mean Throughput,desc_sort_geonameid,60.90,ops/s +Median Throughput,desc_sort_geonameid,60.90,ops/s +Max Throughput,desc_sort_geonameid,60.90,ops/s +100th percentile latency,desc_sort_geonameid,26.10080299996298,ms +100th percentile service time,desc_sort_geonameid,9.38935399994989,ms +error rate,desc_sort_geonameid,0.00,% +Min Throughput,desc_sort_with_after_geonameid,76.25,ops/s +Mean Throughput,desc_sort_with_after_geonameid,76.25,ops/s +Median Throughput,desc_sort_with_after_geonameid,76.25,ops/s +Max Throughput,desc_sort_with_after_geonameid,76.25,ops/s +100th percentile latency,desc_sort_with_after_geonameid,26.196819000006144,ms +100th percentile service time,desc_sort_with_after_geonameid,12.640498999985539,ms +error rate,desc_sort_with_after_geonameid,0.00,% +Min Throughput,asc_sort_geonameid,65.13,ops/s +Mean Throughput,asc_sort_geonameid,65.13,ops/s +Median Throughput,asc_sort_geonameid,65.13,ops/s +Max Throughput,asc_sort_geonameid,65.13,ops/s +100th percentile latency,asc_sort_geonameid,23.98946799996793,ms +100th percentile service time,asc_sort_geonameid,8.35734199995386,ms +error rate,asc_sort_geonameid,0.00,% +Min Throughput,asc_sort_with_after_geonameid,110.72,ops/s +Mean Throughput,asc_sort_with_after_geonameid,110.72,ops/s +Median Throughput,asc_sort_with_after_geonameid,110.72,ops/s +Max Throughput,asc_sort_with_after_geonameid,110.72,ops/s +100th percentile latency,asc_sort_with_after_geonameid,13.891357000034077,ms +100th percentile service time,asc_sort_with_after_geonameid,4.56060500005151,ms +error rate,asc_sort_with_after_geonameid,0.00,% +Metric,Task,Value,Unit +Cumulative indexing time of primary shards,,0.0036666666666666666,min +Min cumulative indexing time across primary shards,,0.0005166666666666667,min +Median cumulative indexing time across primary shards,,0.0007833333333333334,min +Max cumulative indexing time across primary shards,,0.0008333333333333334,min +Cumulative indexing throttle time of primary shards,,0,min +Min cumulative indexing throttle time across primary shards,,0,min +Median cumulative indexing throttle time across primary shards,,0,min +Max cumulative indexing throttle time across primary shards,,0,min +Cumulative merge time of primary shards,,0,min +Cumulative merge count of primary shards,,0, +Min cumulative merge time across primary shards,,0,min +Median cumulative merge time across primary shards,,0,min +Max cumulative merge time across primary shards,,0,min +Cumulative merge throttle time of primary shards,,0,min +Min cumulative merge throttle time across primary shards,,0,min +Median cumulative merge throttle time across primary shards,,0,min +Max cumulative merge throttle time across primary shards,,0,min +Cumulative refresh time of primary shards,,0.0024333333333333334,min +Cumulative refresh count of primary shards,,30, +Min cumulative refresh time across primary shards,,0.00035,min +Median cumulative refresh time across primary shards,,0.0005166666666666667,min +Max cumulative refresh time across primary shards,,0.0006,min +Cumulative flush time of primary shards,,0,min +Cumulative flush count of primary shards,,0, +Min cumulative flush time across primary shards,,0,min +Median cumulative flush time across primary shards,,0,min +Max cumulative flush time across primary shards,,0,min +Total Young Gen GC time,,0,s +Total Young Gen GC count,,0, +Total Old Gen GC time,,0,s +Total Old Gen GC count,,0, +Dataset size,,0.0003236671909689903,GB +Store size,,0.0003236671909689903,GB +Translog size,,2.561137080192566e-07,GB +Heap used for segments,,0,MB +Heap used for doc values,,0,MB +Heap used for terms,,0,MB +Heap used for norms,,0,MB +Heap used for points,,0,MB +Heap used for stored fields,,0,MB +Segment count,,5, +Total Ingest Pipeline count,,0, +Total Ingest Pipeline time,,0,s +Total Ingest Pipeline failed,,0, +Min Throughput,index-append,9476.96,docs/s +Mean Throughput,index-append,9476.96,docs/s +Median Throughput,index-append,9476.96,docs/s +Max Throughput,index-append,9476.96,docs/s +50th percentile latency,index-append,135.9809470000073,ms +100th percentile latency,index-append,198.33244400001604,ms +50th percentile service time,index-append,135.9809470000073,ms +100th percentile service time,index-append,198.33244400001604,ms +error rate,index-append,0.00,% +Min Throughput,index-stats,120.09,ops/s +Mean Throughput,index-stats,120.09,ops/s +Median Throughput,index-stats,120.09,ops/s +Max Throughput,index-stats,120.09,ops/s +100th percentile latency,index-stats,12.128651000011814,ms +100th percentile service time,index-stats,3.522139999972751,ms +error rate,index-stats,0.00,% +Min Throughput,node-stats,61.95,ops/s +Mean Throughput,node-stats,61.95,ops/s +Median Throughput,node-stats,61.95,ops/s +Max Throughput,node-stats,61.95,ops/s +100th percentile latency,node-stats,24.29407399995398,ms +100th percentile service time,node-stats,7.514713999967171,ms +error rate,node-stats,0.00,% +Min Throughput,default,61.50,ops/s +Mean Throughput,default,61.50,ops/s +Median Throughput,default,61.50,ops/s +Max Throughput,default,61.50,ops/s +100th percentile latency,default,22.132435999992595,ms +100th percentile service time,default,5.592903999968257,ms +error rate,default,0.00,% +Min Throughput,term,139.78,ops/s +Mean Throughput,term,139.78,ops/s +Median Throughput,term,139.78,ops/s +Max Throughput,term,139.78,ops/s +100th percentile latency,term,11.871426999960022,ms +100th percentile service time,term,3.8017339999214528,ms +error rate,term,0.00,% +Min Throughput,phrase,114.15,ops/s +Mean Throughput,phrase,114.15,ops/s +Median Throughput,phrase,114.15,ops/s +Max Throughput,phrase,114.15,ops/s +100th percentile latency,phrase,14.49915399996371,ms +100th percentile service time,phrase,5.318823999914457,ms +error rate,phrase,0.00,% +Min Throughput,country_agg_uncached,69.79,ops/s +Mean Throughput,country_agg_uncached,69.79,ops/s +Median Throughput,country_agg_uncached,69.79,ops/s +Max Throughput,country_agg_uncached,69.79,ops/s +100th percentile latency,country_agg_uncached,23.401547999924333,ms +100th percentile service time,country_agg_uncached,8.792318000018895,ms +error rate,country_agg_uncached,0.00,% +Min Throughput,country_agg_cached,76.39,ops/s +Mean Throughput,country_agg_cached,76.39,ops/s +Median Throughput,country_agg_cached,76.39,ops/s +Max Throughput,country_agg_cached,76.39,ops/s +100th percentile latency,country_agg_cached,22.500943999943956,ms +100th percentile service time,country_agg_cached,9.051519999957236,ms +error rate,country_agg_cached,0.00,% +Min Throughput,scroll,47.53,pages/s +Mean Throughput,scroll,47.53,pages/s +Median Throughput,scroll,47.53,pages/s +Max Throughput,scroll,47.53,pages/s +100th percentile latency,scroll,82.65710499995294,ms +100th percentile service time,scroll,39.878026999986105,ms +error rate,scroll,0.00,% +Min Throughput,expression,89.08,ops/s +Mean Throughput,expression,89.08,ops/s +Median Throughput,expression,89.08,ops/s +Max Throughput,expression,89.08,ops/s +100th percentile latency,expression,19.51869499998793,ms +100th percentile service time,expression,7.728913000050852,ms +error rate,expression,0.00,% +Min Throughput,painless_static,76.50,ops/s +Mean Throughput,painless_static,76.50,ops/s +Median Throughput,painless_static,76.50,ops/s +Max Throughput,painless_static,76.50,ops/s +100th percentile latency,painless_static,25.17671700002211,ms +100th percentile service time,painless_static,11.82104800000161,ms +error rate,painless_static,0.00,% +Min Throughput,painless_dynamic,81.07,ops/s +Mean Throughput,painless_dynamic,81.07,ops/s +Median Throughput,painless_dynamic,81.07,ops/s +Max Throughput,painless_dynamic,81.07,ops/s +100th percentile latency,painless_dynamic,21.390404999920065,ms +100th percentile service time,painless_dynamic,8.589051999933872,ms +error rate,painless_dynamic,0.00,% +Min Throughput,decay_geo_gauss_function_score,109.06,ops/s +Mean Throughput,decay_geo_gauss_function_score,109.06,ops/s +Median Throughput,decay_geo_gauss_function_score,109.06,ops/s +Max Throughput,decay_geo_gauss_function_score,109.06,ops/s +100th percentile latency,decay_geo_gauss_function_score,15.251723999995193,ms +100th percentile service time,decay_geo_gauss_function_score,5.770663999896897,ms +error rate,decay_geo_gauss_function_score,0.00,% +Min Throughput,decay_geo_gauss_script_score,105.08,ops/s +Mean Throughput,decay_geo_gauss_script_score,105.08,ops/s +Median Throughput,decay_geo_gauss_script_score,105.08,ops/s +Max Throughput,decay_geo_gauss_script_score,105.08,ops/s +100th percentile latency,decay_geo_gauss_script_score,17.79644099997313,ms +100th percentile service time,decay_geo_gauss_script_score,7.966606999957548,ms +error rate,decay_geo_gauss_script_score,0.00,% +Min Throughput,field_value_function_score,116.19,ops/s +Mean Throughput,field_value_function_score,116.19,ops/s +Median Throughput,field_value_function_score,116.19,ops/s +Max Throughput,field_value_function_score,116.19,ops/s +100th percentile latency,field_value_function_score,14.850521000084882,ms +100th percentile service time,field_value_function_score,5.726571000082004,ms +error rate,field_value_function_score,0.00,% +Min Throughput,field_value_script_score,93.56,ops/s +Mean Throughput,field_value_script_score,93.56,ops/s +Median Throughput,field_value_script_score,93.56,ops/s +Max Throughput,field_value_script_score,93.56,ops/s +100th percentile latency,field_value_script_score,21.80660099998022,ms +100th percentile service time,field_value_script_score,10.821197000041138,ms +error rate,field_value_script_score,0.00,% +Min Throughput,large_terms,6.89,ops/s +Mean Throughput,large_terms,6.89,ops/s +Median Throughput,large_terms,6.89,ops/s +Max Throughput,large_terms,6.89,ops/s +100th percentile latency,large_terms,278.7682100000666,ms +100th percentile service time,large_terms,127.296881999996,ms +error rate,large_terms,0.00,% +Min Throughput,large_filtered_terms,12.08,ops/s +Mean Throughput,large_filtered_terms,12.08,ops/s +Median Throughput,large_filtered_terms,12.08,ops/s +Max Throughput,large_filtered_terms,12.08,ops/s +100th percentile latency,large_filtered_terms,163.45256400006747,ms +100th percentile service time,large_filtered_terms,73.9418680000199,ms +error rate,large_filtered_terms,0.00,% +Min Throughput,large_prohibited_terms,8.65,ops/s +Mean Throughput,large_prohibited_terms,8.65,ops/s +Median Throughput,large_prohibited_terms,8.65,ops/s +Max Throughput,large_prohibited_terms,8.65,ops/s +100th percentile latency,large_prohibited_terms,223.2875120000699,ms +100th percentile service time,large_prohibited_terms,101.53825800000504,ms +error rate,large_prohibited_terms,0.00,% +Min Throughput,desc_sort_population,107.80,ops/s +Mean Throughput,desc_sort_population,107.80,ops/s +Median Throughput,desc_sort_population,107.80,ops/s +Max Throughput,desc_sort_population,107.80,ops/s +100th percentile latency,desc_sort_population,15.771683000025405,ms +100th percentile service time,desc_sort_population,6.201829000019643,ms +error rate,desc_sort_population,0.00,% +Min Throughput,asc_sort_population,123.34,ops/s +Mean Throughput,asc_sort_population,123.34,ops/s +Median Throughput,asc_sort_population,123.34,ops/s +Max Throughput,asc_sort_population,123.34,ops/s +100th percentile latency,asc_sort_population,16.886569000007512,ms +100th percentile service time,asc_sort_population,8.308684999974503,ms +error rate,asc_sort_population,0.00,% +Min Throughput,asc_sort_with_after_population,101.53,ops/s +Mean Throughput,asc_sort_with_after_population,101.53,ops/s +Median Throughput,asc_sort_with_after_population,101.53,ops/s +Max Throughput,asc_sort_with_after_population,101.53,ops/s +100th percentile latency,asc_sort_with_after_population,13.655803999995442,ms +100th percentile service time,asc_sort_with_after_population,3.506921999928636,ms +error rate,asc_sort_with_after_population,0.00,% +Min Throughput,desc_sort_geonameid,66.44,ops/s +Mean Throughput,desc_sort_geonameid,66.44,ops/s +Median Throughput,desc_sort_geonameid,66.44,ops/s +Max Throughput,desc_sort_geonameid,66.44,ops/s +100th percentile latency,desc_sort_geonameid,23.736659999940457,ms +100th percentile service time,desc_sort_geonameid,8.313776000022699,ms +error rate,desc_sort_geonameid,0.00,% +Min Throughput,desc_sort_with_after_geonameid,114.98,ops/s +Mean Throughput,desc_sort_with_after_geonameid,114.98,ops/s +Median Throughput,desc_sort_with_after_geonameid,114.98,ops/s +Max Throughput,desc_sort_with_after_geonameid,114.98,ops/s +100th percentile latency,desc_sort_with_after_geonameid,15.178980999962732,ms +100th percentile service time,desc_sort_with_after_geonameid,6.037147999904846,ms +error rate,desc_sort_with_after_geonameid,0.00,% +Min Throughput,asc_sort_geonameid,117.96,ops/s +Mean Throughput,asc_sort_geonameid,117.96,ops/s +Median Throughput,asc_sort_geonameid,117.96,ops/s +Max Throughput,asc_sort_geonameid,117.96,ops/s +100th percentile latency,asc_sort_geonameid,14.10403599993515,ms +100th percentile service time,asc_sort_geonameid,5.3342209999982515,ms +error rate,asc_sort_geonameid,0.00,% +Min Throughput,asc_sort_with_after_geonameid,162.40,ops/s +Mean Throughput,asc_sort_with_after_geonameid,162.40,ops/s +Median Throughput,asc_sort_with_after_geonameid,162.40,ops/s +Max Throughput,asc_sort_with_after_geonameid,162.40,ops/s +100th percentile latency,asc_sort_with_after_geonameid,10.568196999997781,ms +100th percentile service time,asc_sort_with_after_geonameid,4.075862000036068,ms +error rate,asc_sort_with_after_geonameid,0.00,% \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyClientExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyClientExecutor.cs new file mode 100644 index 0000000000..4ab8c04ce1 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyClientExecutor.cs @@ -0,0 +1,427 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Linq; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + using VirtualClient.Contracts.Metadata; + + /// + /// The Elasticsearch Rally Client workload executor. + /// + public class ElasticsearchRallyClientExecutor : ElasticsearchRallyExecutor + { + /// + /// Initializes a new instance of the class. + /// + /// An enumeration of dependencies that can be used for dependency injection. + /// An enumeration of key-value pairs that can control the execution of the component. + public ElasticsearchRallyClientExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + } + + /// + /// The Rally Distribution Version. + /// If not specified, the Rally latest version will be used. + /// + public string RallyVersion + { + get + { + return this.Parameters.GetValue(nameof(ElasticsearchRallyClientExecutor.RallyVersion)); + } + } + + /// + /// The track targeted for run by Rally. + /// + public string RallyTrackName + { + get + { + return this.Parameters.GetValue(nameof(ElasticsearchRallyClientExecutor.RallyTrackName)); + } + } + + /// + /// Command arguments to control Rally + /// https://esrally.readthedocs.io/en/stable/command_line_reference.html + /// + public string RallyCommandLineArguments + { + get + { + return this.Parameters.GetValue(nameof(ElasticsearchRallyClientExecutor.RallyCommandLineArguments), string.Empty); + } + } + + /// + /// Executes the workload. + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override async Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + await this.Logger.LogMessageAsync($"{this.TypeName}.ExecuteClient", telemetryContext.Clone(), async () => + { + ElasticsearchRallyState state = await this.StateManager.GetStateAsync(nameof(ElasticsearchRallyState), cancellationToken) + ?? new ElasticsearchRallyState(); + + if (cancellationToken.IsCancellationRequested) + { + return; + } + + ClientInstance clientInstance = this.GetLayoutClientInstances(ClientRole.Server).FirstOrDefault(); + + if ( + clientInstance == null || + string.IsNullOrEmpty(clientInstance.IPAddress)) + { + throw new WorkloadException( + $"Elasticsearch Rally Client IP Address must be defined.", + ErrorReason.LayoutInvalid); + } + + string targetHost = clientInstance?.IPAddress; + if (string.IsNullOrEmpty(targetHost)) + { + throw new WorkloadException( + $"Elasticsearch Rally Client could not determine the target host from the layout.", + ErrorReason.LayoutInvalid); + } + + string user = this.PlatformSpecifics.GetLoggedInUser(); + int port = this.Port; + string trackName = this.RallyTrackName; + string dataDirectory = await this.GetDataDirectoryAsync(cancellationToken); + + string rallySharedStoragePath = $"{dataDirectory}/esrally"; // Used for large, shareable, reusable data + string rallyUserHomePath = $"/home/{user}"; // Used for user‑specific results and metadata + + if (!state.RallyConfigured) + { + this.StartRallyClient( + user, + targetHost, + port, + trackName, + rallySharedStoragePath, + rallyUserHomePath, + telemetryContext.Clone(), + cancellationToken); + + if (!cancellationToken.IsCancellationRequested) + { + state.RallyConfigured = true; + await this.StateManager.SaveStateAsync(nameof(ElasticsearchRallyState), state, cancellationToken); + } + } + + this.CleanupElasticSearchCluster( + telemetryContext.Clone(), + cancellationToken, + targetHost, + port); + + this.RunRallyClient( + telemetryContext.Clone(), + cancellationToken, + user, + targetHost, + port, + trackName, + rallySharedStoragePath, + rallyUserHomePath); + }); + + return; + } + + /// + /// Reads the contents of the specified report file and returns it as a single string. + /// + /// The full path to the report file to read. Cannot be null or an empty string. + /// The contents of the report file as a single string. + protected virtual string ReadReportFile(string reportPath) + { + return System.IO.File.ReadAllText(reportPath); + } + + /// + /// Checks whether the specified server is available by attempting to connect to the given host and port. + /// + /// This method waits up to timeout seconds before performing the availability check. Override + /// this method to customize the server availability check logic in derived classes. + /// The context for telemetry and logging associated with this operation. + /// The cancellation token to observe while waiting for the server to become available. + /// The DNS name or IP address of the server to check for availability. Cannot be null or empty. + /// The network port number on the target server to check. Must be a valid TCP port number. + /// The amount of time in milliseconds to wait before performing the availability check. Default is 60000 ms. + /// true if the server at the specified host and port is available; otherwise, false. + protected virtual bool CheckServerAvailable( + EventContext telemetryContext, + CancellationToken cancellationToken, + string targetHost, + int port, + int timeout = 60000) + { + Thread.Sleep(timeout); // wait for server to be available + + return this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyUrlServerCall", $"curl {targetHost}:{port}"); + } + + private void StartRallyClient( + string user, + string targetHost, + int port, + string trackName, + string rallySharedStoragePath, + string rallyUserHomePath, + EventContext telemetryContext, + CancellationToken cancellationToken) + { + // install es rally + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyCheckPyhton3", $"python3 --version", true); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyCheckPip3", $"pip3 --version", true); + + // using pipx to install esrally, prepare the environment and avoid dependency conflicts + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallySetPixPathRoot", $"pipx ensurepath", true); + + string esRallyInstallCommand = "pipx install esrally"; + if (!string.IsNullOrEmpty(this.RallyVersion)) + { + esRallyInstallCommand = $"{esRallyInstallCommand}=={this.RallyVersion}"; + } + + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyInstall", esRallyInstallCommand, true); + + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyMakeSharedStorage", $"mkdir -p {rallySharedStoragePath}"); + + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyChown", $"chown -R {user}:{user} {rallySharedStoragePath}", true); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallySharedStorageCheck", $"ls -ld {rallySharedStoragePath}", true); + this.RunCommandAsUser(telemetryContext, cancellationToken, user, "RallyUserTouch", $"echo ok > {rallySharedStoragePath}/test.txt", true); + + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyChownUserHome", $"chown -R {user}:{user} {rallyUserHomePath}"); + + this.RunESRallyCommand(telemetryContext, cancellationToken, user, rallyUserHomePath, rallySharedStoragePath, "RallyCheckEsrallyCheck", "--version"); + this.RunESRallyCommand(telemetryContext, cancellationToken, user, rallyUserHomePath, rallySharedStoragePath, "RallyInfo", $"info --track={trackName}"); + this.RunESRallyCommand(telemetryContext, cancellationToken, user, rallyUserHomePath, rallySharedStoragePath, "RallyListTracks", "list tracks"); + + // client environment is ready, now we can connect to the Elasticsearch server + + int tries = 0; + while (!this.CheckServerAvailable(telemetryContext, cancellationToken, targetHost, port)) + { + int limit = 20; + if (tries++ >= limit) + { + throw new WorkloadException( + $"Elasticsearch Rally Client could not reach the server at {targetHost} after {limit} attempts.", + ErrorReason.WorkloadFailed); + } + + this.Logger.LogMessage($"{this.TypeName}.ElasticsearchServerConnectionAttempt-{tries}-of-{limit}", telemetryContext); + } + + this.Logger.LogMessage($"{this.TypeName}.ElasticsearchServerIsReady", telemetryContext); + } + + private void CleanupElasticSearchCluster(EventContext telemetryContext, CancellationToken cancellationToken, string targetHost, int port) + { + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyCleanupElasticSearch", $"curl -X DELETE {targetHost}:{port}/_all"); + } + + private void RunRallyClient( + EventContext telemetryContext, + CancellationToken cancellationToken, + string user, + string targetHost, + int port, + string trackName, + string rallySharedStoragePath, + string rallyUserHomePath) + { + DateTime start = DateTime.Now; + string raceId = Guid.NewGuid().ToString(); + string reportPath = $"{rallySharedStoragePath}/report.csv"; + + if (this.CheckFileExists(reportPath)) + { + this.RunCommandAsRoot(telemetryContext, cancellationToken, "RallyRemoveOldReport", $"rm -f {reportPath}", false); + } + + string rallyCommand = this.BuildRallyCommandLineArguments( + trackName, + raceId, + targetHost, + port, + reportPath); + + this.RunESRallyCommand(telemetryContext, cancellationToken, user, rallyUserHomePath, rallySharedStoragePath, "RallyExecution", rallyCommand); + + this.RunESRallyCommand(telemetryContext, cancellationToken, user, rallyUserHomePath, rallySharedStoragePath, "RallyListRaces", "list races"); + + // race.json is undocumented and not present in esrally 2.5.0 and later versions by default, so we cannot depend on it. + string resultsPath = $"{rallySharedStoragePath}/.rally/benchmarks/races/{raceId}/race.json"; + telemetryContext.AddContext("RallyResultsJsonPath", resultsPath); + telemetryContext.AddContext("RallyReportCsvPath", reportPath); + + if (!this.CheckFileExists(reportPath)) + { + throw new WorkloadException( + $"{this.TypeName}.RallyReportCsvMissing", + ErrorReason.WorkloadUnexpectedAnomaly); + } + else + { + try + { + this.CaptureMetrics(reportPath, rallyCommand, raceId, start, DateTime.Now, telemetryContext, cancellationToken); + } + catch (Exception ex) + { + throw new WorkloadException( + $"{this.TypeName}.RallyReportCsvFailed", + ex, + ErrorReason.WorkloadUnexpectedAnomaly); + } + } + } + + private string BuildRallyCommandLineArguments( + string trackName, + string raceId, + string targetHost, + int port, + string reportPath) + { + // using report-file command line to capture results in CSV format + // https://esrally.readthedocs.io/en/stable/command_line_reference.html#report-file + + string rallyCommand = string.Concat( + "race ", + $"--track={trackName} ", + $"--race-id={raceId} ", + $"--target-hosts={targetHost}:{port} ", + $"--show-in-report=available ", // all, all-percentiles, available : using available because "all" thrown null type error + $"--report-format=csv ", + $"--report-file={reportPath} ", + $"--pipeline=benchmark-only ", + $"--runtime-jdk=bundled"); + + if (!string.IsNullOrEmpty(this.RallyCommandLineArguments)) + { + rallyCommand = string.Concat(rallyCommand, " ", this.RallyCommandLineArguments); + } + + return rallyCommand; + } + + private void RunESRallyCommand(EventContext telemetryContext, CancellationToken cancellationToken, string user, string rallyUserHomePath, string rallySharedStoragePath, string key, string esRallyCommand) + { + // hey points of this solution: + // - avoids dotfiles which are very cumbersome to deal with .net process + // - wrapper script quirks by calling python3 -m + // - inlines all required esrally environment arguments via env. + + var pipxBin = $"{rallyUserHomePath}/.local/bin"; + + // Build PATH deterministically (prepend pipx bin) + var basePath = Environment.GetEnvironmentVariable("PATH") ?? "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"; + var childPath = $"{pipxBin}:{basePath}"; + + string shellCommand = string.Concat( + $"-u {user} -H ", // set user scope + "env ", // set environment arguments for current process session + $"HOME={rallyUserHomePath} ", // user storage + $"RALLY_HOME={rallySharedStoragePath} ", // shared storage + $"XDG_STATE_HOME={rallyUserHomePath}/.local/state ", + $"PATH={childPath} ", + "python3 -m pipx run esrally ", // esrally lives inside the pipx venv, not system Python. + esRallyCommand); + + this.RunCommandAsRoot(telemetryContext, cancellationToken, key, shellCommand, true); + } + + private void CaptureMetrics(string reportPath, string rallyExecutionArguments, string raceId, DateTime startTime, DateTime exitTime, EventContext telemetryContext, CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return; + } + + string reportContents = this.ReadReportFile(reportPath); + + ElasticsearchRallyMetricsParser elasticsearchRallyMetricsParser = new ElasticsearchRallyMetricsParser( + reportContents, + new Dictionary + { + [nameof(this.ElasticsearchVersion)] = this.ElasticsearchVersion, + [nameof(this.RallyVersion)] = this.RallyVersion ?? "latest", + [nameof(this.RallyTrackName)] = this.RallyTrackName, + [nameof(raceId)] = raceId, + }, + false); + + string[] reportLines = elasticsearchRallyMetricsParser.ReportLines; + + if (reportLines.Length < 2) + { + this.Logger.LogMessage($"{this.TypeName}.RallyReportCsvInsufficientData", telemetryContext); + return; + } + + telemetryContext.AddContext("RallyReportCsvContents", reportContents.Take(5)); + this.Logger.LogMessage($"{this.TypeName}.RallyReportCsv", telemetryContext); + + this.MetadataContract.AddForScenario( + "ElasticsearchRally", + rallyExecutionArguments, + toolVersion: null); + + this.MetadataContract.Apply(telemetryContext); + + if (reportLines.Length > 0) + { + try + { + IList metrics = elasticsearchRallyMetricsParser.Parse(); + + if (this.MetricFilters?.Any() == true) + { + metrics = metrics.FilterBy(this.MetricFilters).ToList(); + } + + this.Logger.LogMetrics( + toolName: "ElasticsearchRally", + scenarioName: this.MetricScenario ?? this.Scenario, + startTime, + exitTime, + metrics, + null, + scenarioArguments: rallyExecutionArguments, + this.Tags, + telemetryContext); + } + catch (Exception exc) + { + throw new WorkloadException( + $"Capture metrics failed.", + exc, + ErrorReason.InvalidResults); + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyExecutor.cs new file mode 100644 index 0000000000..7e646ab203 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyExecutor.cs @@ -0,0 +1,331 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Linq; + using System.Net; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient; + using VirtualClient.Common; + using VirtualClient.Common.Extensions; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// Base class for all Elasticsearch Rally workload executors. + /// + [SupportedPlatforms("linux-arm64,linux-x64,win-arm64,win-x64")] + public abstract class ElasticsearchRallyExecutor : VirtualClientComponent + { + /// + /// Constructor for + /// + /// Provides required dependencies to the component. + /// Parameters defined in the profile or supplied on the command line. + public ElasticsearchRallyExecutor(IServiceCollection dependencies, IDictionary parameters) + : base(dependencies, parameters) + { + this.SupportedRoles = new List + { + ClientRole.Client, + ClientRole.Server + }; + } + + /// + /// The Elasticsearch Distribution Version. + /// + public string ElasticsearchVersion + { + get + { + return this.Parameters.GetValue(nameof(ElasticsearchRallyServerExecutor.ElasticsearchVersion), "9.2.3"); + } + } + + /// + /// Disk filter specified + /// + public string DiskFilter + { + get + { + return this.Parameters.GetValue(nameof(this.DiskFilter), "osdisk:false&biggestsize"); + } + } + + /// + /// Elasticsearch Node Port Number + /// + public int Port + { + get + { + return this.Parameters.GetValue(nameof(this.Port), 9200); + } + } + + /// + /// Manages the state of the system. + /// + protected IStateManager StateManager => this.Dependencies.GetService(); + + /// + /// Initializes the environment for execution of the Rally workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + if (!this.IsMultiRoleLayout()) + { + throw new WorkloadException( + $"{this.PackageName} Client/Server requires at least 2 nodes to run", + ErrorReason.LayoutInvalid); + } + + if (!cancellationToken.IsCancellationRequested) + { + IApiClientManager clientManager = this.Dependencies.GetService(); + + ClientInstance instance = this.GetLayoutClientInstance(); + string layoutIPAddress = instance.IPAddress; + + this.ThrowIfLayoutClientIPAddressNotFound(layoutIPAddress); + this.ThrowIfRoleNotSupported(instance.Role); + + ClientInstance clientInstance = this.GetLayoutClientInstances(ClientRole.Client).First(); + + IPAddress.TryParse(clientInstance.IPAddress, out IPAddress clientIpAddress); + telemetryContext.AddContext("ClientIpAddress", clientIpAddress.ToString()); + + IEnumerable serverInstances = this.GetLayoutClientInstances(ClientRole.Server); + + foreach (ClientInstance serverInstance in serverInstances) + { + IPAddress.TryParse(serverInstance.IPAddress, out IPAddress serverIPAddress); + + IApiClient apiClient = clientManager.GetOrCreateApiClient(serverIPAddress.ToString(), serverIPAddress); + this.RegisterToSendExitNotifications($"{this.TypeName}.ExitNotification", apiClient); + } + } + + await Task.CompletedTask; + + return; + } + + /// + /// Get filtered data directory + /// + /// + /// + /// + protected virtual async Task GetDataDirectoryAsync(CancellationToken cancellationToken) + { + if (cancellationToken.IsCancellationRequested) + { + return null; + } + + ISystemManagement systemManagement = this.Dependencies.GetService(); + + IEnumerable disks = await systemManagement.DiskManager.GetDisksAsync(cancellationToken); + + if (disks?.Any() != true) + { + throw new WorkloadException( + "Unexpected scenario. The disks defined for the system could not be properly enumerated.", + ErrorReason.WorkloadUnexpectedAnomaly); + } + + IEnumerable disksToTest = DiskFilters.FilterDisks(disks, this.DiskFilter, this.Platform).ToList(); + + if (disksToTest?.Any() != true) + { + throw new WorkloadException( + "Expected disks to test not found. Given the parameters defined for the profile action/step or those passed " + + "in on the command line, the requisite disks do not exist on the system or could not be identified based on the properties " + + "of the existing disks.", + ErrorReason.DependencyNotFound); + } + + return $"{disksToTest.First().GetPreferredAccessPath(this.Platform)}"; + } + + /// + /// Determines whether a file exists at the specified path. + /// + /// The method does not check whether the caller has permission to access the file. + /// Passing an invalid path may result in false being returned. + /// The path to the file to check. This can be either a relative or an absolute path. + /// true if a file exists at the specified path; otherwise, false. + protected virtual bool CheckFileExists(string path) + { + return File.Exists(path); + } + + /// + /// Determines whether the specified directory exists. + /// + /// The path to the directory to check. + /// true if the directory exists; otherwise, false. + protected virtual bool CheckDirectoryExists(string path) + { + return Directory.Exists(path); + } + + /// + /// Copies a file from the source path to the destination path. + /// + /// The path of the file to copy. + /// The destination path where the file should be copied. + /// Whether to overwrite the file if it already exists at the destination. + protected virtual void FileCopy(string sourcePath, string destinationPath, bool overwrite) + { + File.Copy(sourcePath, destinationPath, overwrite); + } + + /// + /// Creates a directory at the specified path. + /// + /// The directory to create + protected virtual void CreateDirectory(string path) + { + Directory.CreateDirectory(path); + } + + /// + /// Runs a bash script command. + /// + /// + /// + /// Task identifier + /// + /// + /// + protected bool RunCommandScript(EventContext telemetryContext, CancellationToken cancellationToken, string key, string script, bool throwOnError = false) + { + bool ok = this.RunCommand(telemetryContext, cancellationToken, "/bin/bash", BuildBashScript(script), out string output, out string error); + + this.HandleTelemetry(telemetryContext, key, script, throwOnError, ok, output, error); + + return ok; + } + + /// + /// Runs a command as root. + /// + /// + /// + /// Task identifier + /// + /// + /// + protected bool RunCommandAsRoot(EventContext telemetryContext, CancellationToken cancellationToken, string key, string command, bool throwOnError = false) + { + return this.RunCommandAsUser(telemetryContext, cancellationToken, null, key, command, throwOnError); + } + + /// + /// Runs a command as a specific user. + /// + /// + /// + /// + /// + /// + /// + /// + protected bool RunCommandAsUser(EventContext telemetryContext, CancellationToken cancellationToken, string user, string command, out string output, out string error) + { + return + this.RunCommand( + telemetryContext, + cancellationToken, + "/usr/bin/sudo", + string.IsNullOrEmpty(user) ? command : $"-u {user} -H bash {BuildBashScript(command)}", + out output, + out error); + } + + /// + /// Runs a command as a specific user. + /// + /// + /// + /// + /// Task identifier + /// + /// + /// + /// + protected bool RunCommandAsUser(EventContext telemetryContext, CancellationToken cancellationToken, string user, string key, string command, bool throwOnError = false) + { + bool ok = this.RunCommandAsUser(telemetryContext, cancellationToken, user, command, out string output, out string error); + + this.HandleTelemetry(telemetryContext, key, command, throwOnError, ok, output, error); + + return ok; + } + + /// + /// Runs a command. + /// + /// + /// + /// + /// + /// + /// + /// + protected virtual bool RunCommand(EventContext telemetryContext, CancellationToken cancellationToken, string command, string arguments, out string output, out string error) + { + output = null; + error = null; + + try + { + IProcessProxy p = this.ExecuteCommandAsync(command, arguments, null, telemetryContext, cancellationToken).Result; + output = p.StandardOutput.ToString().Trim(); + error = p.StandardError.ToString().Trim(); + return p.ExitCode == 0; + } + catch (Exception ex) + { + error = ex.ToString(); + return false; + } + } + + private static string BuildBashScript(string script) + { + return string.Concat("-lc \"", script.Replace("\"", "\\\""), "\""); + } + + private void HandleTelemetry(EventContext telemetryContext, string key, string command, bool throwOnError, bool ok, string output, string error) + { + telemetryContext.AddContext($"{this.TypeName}.{key}Command", command); + telemetryContext.AddContext($"{this.TypeName}.{key}Output", output); + telemetryContext.AddContext($"{this.TypeName}.{key}Error", error); + telemetryContext.AddContext($"{this.TypeName}.{key}Ok", ok); + this.Logger.LogMessage($"{this.TypeName}.{key}", telemetryContext); + + if (!ok) + { + this.Logger.LogMessage($"{this.TypeName}.{key}Failed", telemetryContext); + + if (throwOnError) + { + throw new WorkloadException( + $"Rally server configuration failed. Output: {output}; Error: {error}", + ErrorReason.WorkloadUnexpectedAnomaly); + } + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyMetricsParser.cs new file mode 100644 index 0000000000..bf11b90888 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyMetricsParser.cs @@ -0,0 +1,307 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.Linq; + using VirtualClient; + using VirtualClient.Contracts; + + /// + /// Elasticsearch Rally metrics parser that parses the raw text output from Elasticsearch Rally and converts it into a list of objects. + /// + public class ElasticsearchRallyMetricsParser : MetricsParser + { + /// + /// Constructor for + /// + /// Text to be parsed. + /// Metadata associated with the metrics. + /// Indicates whether to collect all metrics. + public ElasticsearchRallyMetricsParser(string reportContents, Dictionary metadata, bool rallyCollectAllMetrics) + : base(reportContents) + { + if (string.IsNullOrEmpty(reportContents)) + { + throw new ArgumentNullException(nameof(reportContents), "Report contents cannot be null."); + } + + if (metadata == null) + { + throw new ArgumentNullException(nameof(metadata)); + } + + this.ReportLines = reportContents.Split(new[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); + + foreach (var item in metadata) + { + this.Metadata[item.Key] = item.Value; + } + + this.RallyCollectAllMetrics = rallyCollectAllMetrics; + } + + /// + /// Indicates whether to collect all metrics. + /// + public bool RallyCollectAllMetrics { get; private set; } + + /// + /// Allows the user to describe different levels of priority/verbosity to a set of metrics that can + /// be used for queries/filtering. Lower values indicate higher priority. For example, metrics considered + /// to be the most critical for decision making would be set with verbosity = 1 (Critical). + /// + public int MetricsVerbosity + { + get + { + if (this.Metadata.TryGetValue(nameof(this.MetricsVerbosity), out IConvertible verbosityValue) && int.TryParse(verbosityValue.ToString(), out int verbosity)) + { + return verbosity; + } + + return 1; + } + } + + /// + /// The lines of the report to be parsed. + /// + public string[] ReportLines { get; } + + /// + /// Parses the raw text output from Elasticsearch Rally and converts it into a list of objects. + /// + /// A list of objects. + public override IList Parse() + { + return Read(this.ReportLines, this.Metadata, this.RallyCollectAllMetrics, this.MetricsVerbosity); + } + + private static IList Read( + string[] reportLines, + IDictionary metadata, + bool collectAllMetrics, + int metricsVerbosity) + { + if (reportLines == null) + { + throw new ArgumentNullException(nameof(reportLines), "Report lines cannot be null."); + } + + if (metadata == null) + { + throw new ArgumentNullException(nameof(metadata), "Metadata cannot be null."); + } + + IList metrics = new List(); + + foreach (string line in reportLines) + { + // Metric,Task,Value,Unit + string[] cols = line.Split(','); + + if (cols.Length != 4 || !double.TryParse(cols[2], out double value)) + { + continue; + } + + string metricName = cols[0].ToLower(); + string taskName = cols[1].ToLower(); + + // only relevant metrics will be collected + if (!collectAllMetrics && + !IsRelevantMetric(metricName, taskName)) + { + continue; + } + + string unit = cols[3]; + bool isTimeUnit = unit == "s" || unit == "ms" || unit == "min"; + + if (unit.Length == 0 && metricName.Contains($" {MetricNames.Count}")) + { + unit = MetricNames.Count; + } + + MetricRelativity relativity = MetricRelativity.Undefined; + + if (metricName.EndsWith(MetricNames.Throughput)) + { + relativity = MetricRelativity.HigherIsBetter; + } + else if ( + metricName.EndsWith(MetricNames.Latency) || + metricName.EndsWith(MetricNames.ServiceTime) || + metricName.EndsWith(MetricNames.Rate)) + { + relativity = MetricRelativity.LowerIsBetter; + } + + if (relativity == MetricRelativity.Undefined) + { + if (isTimeUnit) + { + relativity = MetricRelativity.LowerIsBetter; + } + } + + metricName = TransformMetricName(metricName); + + if (taskName.Length > 0) + { + metricName = $"{taskName} {metricName}"; + } + + metricName = metricName.Replace(' ', '-'); + + Metric metric = new Metric( + name: metricName, + value: value, + unit: unit, + relativity: relativity, + metadata: metadata); + + if (!CheckVerbosity(metric, metricsVerbosity) && !collectAllMetrics) + { + continue; + } + + metrics.Add(metric); + } + + return metrics; + } + + /// + /// Verbosity levels define a convention for organizing metrics by importance: + /// - 1 (Standard/Critical): Most important metrics for decision making - bandwidth, throughput, IOPS, key latency percentiles (p50, p99) + /// - 2 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 3 (Reserved): Reserved for future expansion + /// - 4 (Reserved): Reserved for future expansion + /// - 5 (Verbose): All diagnostic/internal metrics - histogram buckets, standard deviations, byte counts, I/O counts + /// https://github.com/microsoft/VirtualClient/blob/f1d5410ac2c1cd1acfa6a0901af79cbef1abe9df/src/VirtualClient/VirtualClient.Contracts/Metric.cs#L163 + /// + /// + /// + /// + private static bool CheckVerbosity(Metric metric, int metricsVerbosity) + { + if (metric.Name.EndsWith("P100") || + metric.Name.EndsWith("Mean") || metric.Name.EndsWith("Median") || (metric.Name.Contains("latency") || metric.Name.Contains("throughput"))) + { + metric.Verbosity = 1; // Critical + } + else if (metric.Name.EndsWith("P50") || metric.Name.EndsWith("P90") || metric.Name.EndsWith("P99")) + { + metric.Verbosity = 2; // Detailed + } + else + { + metric.Verbosity = 5; // Verbose + } + + return metric.Verbosity <= metricsVerbosity; + } + + private static bool SwapPrefix(string metricName, string oldPrefix, string newPrefix, out string newMetricName) + { + if (metricName.StartsWith(oldPrefix)) + { + newMetricName = $"{metricName.Substring(oldPrefix.Length).Trim()} {newPrefix}"; + return true; + } + + newMetricName = metricName; + return false; + } + + private static bool SwapPercentileFormat(string metricName, out string newMetricName) + { + // "100th percentile latency" => "latency P100" + + var match = System.Text.RegularExpressions.Regex.Match(metricName, @"^(\d+)th (percentile) (.+)"); + if (match.Success) + { + newMetricName = $"{match.Groups[3].Value} P{match.Groups[1].Value}"; + return true; + } + + newMetricName = metricName; + return false; + } + + private static string TransformMetricName(string metricName) + { + if (SwapPrefix(metricName, "median", "Median", out string newMetricName)) + { + return newMetricName; + } + else if (SwapPrefix(metricName, "mean", "Mean", out newMetricName)) + { + return newMetricName; + } + else if (SwapPercentileFormat(metricName, out newMetricName)) + { + return newMetricName; + } + + return metricName; + } + + private static bool IsRelevantMetric(string metricName, string taskName) + { + if (string.IsNullOrEmpty(taskName)) + { + // summary metrics + return + new string[] + { + "median cumulative indexing time across primary shards", + "median cumulative merge time across primary shards", + "median cumulative refresh time across primary shards", + "median cumulative flush time across primary shards", + "total young gen gc time", + "dataset size", + "translog size", + "segment count", + }.Contains(metricName); + } + else + { + // task metrics + return + new string[] + { + "mean throughput", + "median throughput", + "50th percentile latency", + "90th percentile latency", + "99th percentile latency", + "100th percentile latency", + "50th percentile service time", + "90th percentile service time", + "99th percentile service time", + "100th percentile service time", + "error rate", + }.Contains(metricName); + } + } + + private struct MetricNames + { + public const string Count = "count"; + public const string Latency = "latency"; + public const string Median = "median"; + public const string ServiceTime = "service time"; + public const string P50 = "50th"; + public const string P90 = "90th"; + public const string P99 = "99th"; + public const string P100 = "100th"; + public const string Rate = "rate"; + public const string Throughput = "throughput"; + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyServerExecutor.cs b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyServerExecutor.cs new file mode 100644 index 0000000000..f9cda86db5 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyServerExecutor.cs @@ -0,0 +1,263 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using System.IO; + using System.Threading; + using System.Threading.Tasks; + using Microsoft.Extensions.DependencyInjection; + using VirtualClient; + using VirtualClient.Common.Telemetry; + using VirtualClient.Contracts; + + /// + /// The Elasticsearch Rally Server workload executor. + /// + public class ElasticsearchRallyServerExecutor : ElasticsearchRallyExecutor + { + /// + /// Initializes a new instance of the class. + /// + /// An enumeration of dependencies that can be used for dependency injection. + /// An enumeration of key-value pairs that can control the execution of the component. + public ElasticsearchRallyServerExecutor(IServiceCollection dependencies, IDictionary parameters = null) + : base(dependencies, parameters) + { + } + + /// + /// The timeout duration (in milliseconds) to wait for Elasticsearch to become available after starting. + /// + protected int WaitForElasticsearchAvailabilityTimeout { get; set; } = 30000; + + /// + /// Initializes the environment for execution of the Rally workload. + /// + protected override async Task InitializeAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + await base.InitializeAsync(telemetryContext, cancellationToken) + .ConfigureAwait(false); + + await this.Logger.LogMessageAsync($"{this.TypeName}.ConfigureServer", telemetryContext.Clone(), async () => + { + ElasticsearchRallyState state = await this.StateManager.GetStateAsync(nameof(ElasticsearchRallyState), cancellationToken) + ?? new ElasticsearchRallyState(); + + if (!state.ElasticsearchStarted) + { + await this.StartElasticsearch(telemetryContext, cancellationToken); + + if (!cancellationToken.IsCancellationRequested) + { + state.ElasticsearchStarted = true; + await this.StateManager.SaveStateAsync(nameof(ElasticsearchRallyState), state, cancellationToken); + } + } + }); + } + + /// + /// Executes server side of workload. + /// + /// Provides context information that will be captured with telemetry events. + /// A token that can be used to cancel the operation. + protected override Task ExecuteAsync(EventContext telemetryContext, CancellationToken cancellationToken) + { + return this.Logger.LogMessageAsync($"{nameof(ElasticsearchRallyServerExecutor)}.ExecuteServer", telemetryContext, async () => + { + try + { + this.SetServerOnline(true); + + if (this.IsMultiRoleLayout()) + { + using (BackgroundOperations profiling = BackgroundOperations.BeginProfiling(this, cancellationToken)) + { + await this.WaitAsync(cancellationToken); + } + } + } + finally + { + this.SetServerOnline(false); + } + }); + } + + /// + /// Write all text to a file. + /// + /// The path to the file. + /// The content to write to the file. + protected virtual void WriteAllText(string path, string content) + { + File.WriteAllText(path, content); + } + + /// + /// Read all text from a file. + /// + /// The path to the file. + /// The content of the file. + protected virtual string ReadAllText(string path) + { + return File.ReadAllText(path); + } + + private async Task StartElasticsearch(EventContext telemetryContext, CancellationToken cancellationToken) + { + string mountPoint = await this.GetDataDirectoryAsync(cancellationToken); + + if (this.Platform == PlatformID.Unix) + { + this.StartElasticsearchLinux(telemetryContext, cancellationToken, mountPoint); + } + else + { + throw new NotSupportedException($"The {this.TypeName} does not support execution on {this.Platform} operating system."); + } + } + + private void StartElasticsearchLinux(EventContext telemetryContext, CancellationToken cancellationToken, string mountPoint) + { + string scriptsDirectory = this.PlatformSpecifics.GetScriptPath(this.PackageName.ToLower()); + int port = this.Port; + + telemetryContext.AddContext(nameof(mountPoint), mountPoint); + telemetryContext.AddContext(nameof(scriptsDirectory), scriptsDirectory); + telemetryContext.AddContext(nameof(port), port); + + this.RunCommandAsRoot(telemetryContext, cancellationToken, "SetVmMaxMapCount", "sysctl -w vm.max_map_count=262144"); + + // make the change persistent + this.RunCommandScript(telemetryContext, cancellationToken, "VmMaxMapCountPersist", "echo \"vm.max_map_count=262144\" | sudo tee /etc/sysctl.d/99-elasticsearch.conf"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "VmMaxMapCountSysCtl", "sysctl --system"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "VmMaxMapCountVerify", "sysctl vm.max_map_count"); + + // LimitMEMLOCKinfinity + this.RunCommandScript(telemetryContext, cancellationToken, "LimitMEMLOCKinfinityMkdir", "sudo mkdir -p /etc/systemd/system/elasticsearch.service.d"); + this.RunCommandScript(telemetryContext, cancellationToken, "LimitMEMLOCKinfinityPersist", "printf \"[Service]\nLimitMEMLOCK=infinity\nLimitMEMLOCKSoft=infinity\n\" | sudo tee /etc/systemd/system/elasticsearch.service.d/override.conf"); + + // download and install elasticsearch + this.InstallElasticsearchLinux(telemetryContext, cancellationToken, scriptsDirectory); + + (string rootPath, string dataPath, string logPath) = GetElasticsearchPathsLinux(mountPoint); + + if (!string.IsNullOrEmpty(mountPoint)) + { + // create data and log directories + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchRootPathMkdir", $"mkdir -p {rootPath}"); + + // check if mounted filesystem is read‑only + this.RunCommandScript(telemetryContext, cancellationToken, "CheckMountedFileSystem", $"mount | grep {mountPoint}"); + + // set ownership + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchRepoChown", $"chmod -R 777 {rootPath}", true); + } + + // create elasticsearch.yml + string elasticsearchPathYml = this.CreateElasticsearchYml(telemetryContext, scriptsDirectory, port, dataPath, logPath); + + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchYmlCopy", $"cp {elasticsearchPathYml} /etc/elasticsearch/elasticsearch.yml"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchYml", $"tail -n 10000 /etc/elasticsearch/elasticsearch.yml"); + + // set limits.conf + string limitsConfPath = this.PlatformSpecifics.Combine(scriptsDirectory, "limits.ini"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchLimitsCopy", $"cp {limitsConfPath} /etc/security/limits.conf"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchLimits", $"tail -n 10000 /etc/security/limits.conf"); + + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchRemoveKeystore", "/usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.transport.ssl.keystore.secure_password"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchRemoveTruestore", "/usr/share/elasticsearch/bin/elasticsearch-keystore remove xpack.security.transport.ssl.truststore.secure_password"); + + // run elasticsearch + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchDaemonReexec", "systemctl daemon-reexec"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchEnable", "systemctl enable elasticsearch"); + bool ok = this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchStart", "systemctl start elasticsearch.service"); + Thread.Sleep(30000); // wait for elasticsearch to start + + if (!ok) + { + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchJournal", "journalctl -xeu elasticsearch.service"); + + throw new WorkloadException( + $"Elasticsearch failed to start.", + ErrorReason.WorkloadUnexpectedAnomaly); + } + + // verify elasticsearch is running + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchStatus", "systemctl status elasticsearch.service"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchSocket", "ss -lnt"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchUrlCall", $"curl localhost:{port}"); + } + + private void InstallElasticsearchLinux(EventContext telemetryContext, CancellationToken cancellationToken, string scriptsDirectory) + { + // VirtualClient profile script using LinuxPackageInstallation is throwing a reachability error from Juno deployment: Unable to locate package elasticsearch + // manual installation is not working via wget in VMs deployed by Juno, using Apt instead. + + string elasticsearchMajorVersionKey = $"{this.ElasticsearchVersion.Substring(0, 1)}.x"; + telemetryContext.AddContext(nameof(elasticsearchMajorVersionKey), elasticsearchMajorVersionKey); + + // Elasticsearch documentation here https://www.elastic.co/guide/en/elasticsearch/reference/current/deb.html + this.RunCommandScript(telemetryContext, cancellationToken, "ElasticsearchImport", "curl -fsSL https://artifacts.elastic.co/GPG-KEY-elasticsearch | sudo gpg --dearmor -o /usr/share/keyrings/elasticsearch-keyring.gpg"); + this.RunCommandScript(telemetryContext, cancellationToken, "ElasticsearchAdd", $"echo \"deb [signed-by=/usr/share/keyrings/elasticsearch-keyring.gpg] https://artifacts.elastic.co/packages/{elasticsearchMajorVersionKey}/apt stable main\" | sudo tee /etc/apt/sources.list.d/elastic-{elasticsearchMajorVersionKey}.list"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchUpdate", "apt update"); + this.RunCommandAsRoot(telemetryContext, cancellationToken, "ElasticsearchInstall", "apt install elasticsearch -y"); + } + + private static (string root, string data, string log) GetElasticsearchPathsLinux(string mountPoint) + { + string rootPath; + if (mountPoint == null) + { + rootPath = "/var/elasticsearch"; + } + else + { + rootPath = Path.Combine(mountPoint, "elasticsearch"); + } + + return (rootPath, $"{rootPath}/data", $"{rootPath}/log"); + } + + private string CreateElasticsearchYml(EventContext telemetryContext, string scriptsDirectory, int port, string dataPath, string logPath) + { + string elasticsearchPath = this.PlatformSpecifics.Combine(scriptsDirectory, "elasticsearch.ini"); + if (!this.CheckFileExists(elasticsearchPath)) + { + throw new WorkloadException( + $"The Elasticsearch configuration file (yml) could not be found at the expected path: {elasticsearchPath}", + ErrorReason.WorkloadUnexpectedAnomaly); + } + + Dictionary ymlparameters = new Dictionary() + { + { "port", port.ToString() }, + { "path.data", dataPath }, + { "path.logs", logPath }, + }; + + telemetryContext.AddContext($"{this.TypeName}.{nameof(elasticsearchPath)}", elasticsearchPath); + this.Logger.LogMessage($"{this.TypeName}.ElasticsearchIniRead", telemetryContext); + string elasticsearchYmlContent = this.ReadAllText(elasticsearchPath); + + foreach (KeyValuePair p in ymlparameters) + { + telemetryContext.AddContext($"{this.TypeName}.YmlParameter.{p.Key}", p.Value); + + elasticsearchYmlContent = elasticsearchYmlContent.Replace($"$.parameters.{p.Key}", p.Value); + } + + elasticsearchYmlContent = elasticsearchYmlContent.Replace("$.parameters.port", port.ToString()); + + string elasticsearchPathYml = elasticsearchPath.Replace(".ini", ".yml"); + this.Logger.LogMessage($"{this.TypeName}.ElasticsearchYmlWrite", telemetryContext); + this.WriteAllText(elasticsearchPathYml, elasticsearchYmlContent); + + return elasticsearchPathYml; + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyState.cs b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyState.cs new file mode 100644 index 0000000000..4006f4eea5 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/ElasticsearchRallyState.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +namespace VirtualClient.Actions +{ + using System; + using System.Collections.Generic; + using VirtualClient.Common.Extensions; + using VirtualClient.Contracts; + + internal class ElasticsearchRallyState : State + { + public ElasticsearchRallyState(IDictionary properties = null) + : base(properties) + { + } + + public bool ElasticsearchStarted + { + get + { + return this.Properties.GetValue(nameof(ElasticsearchRallyState.ElasticsearchStarted), false); + } + + set + { + this.Properties[nameof(ElasticsearchRallyState.ElasticsearchStarted)] = value; + } + } + + public bool RallyConfigured + { + get + { + return this.Properties.GetValue(nameof(ElasticsearchRallyState.RallyConfigured), false); + } + + set + { + this.Properties[nameof(ElasticsearchRallyState.RallyConfigured)] = value; + } + } + } +} diff --git a/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/elasticsearch.ini b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/elasticsearch.ini new file mode 100644 index 0000000000..74e9463e89 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/elasticsearch.ini @@ -0,0 +1,122 @@ +# ======================== Elasticsearch Configuration ========================= +# +# NOTE: Elasticsearch comes with reasonable defaults for most settings. +# Before you set out to tweak and tune the configuration, make sure you +# understand what are you trying to accomplish and the consequences. +# +# The primary way of configuring a node is via this file. This template lists +# the most important settings you may want to configure for a production cluster. +# +# Please consult the documentation for further information on configuration options: +# https://www.elastic.co/guide/en/elasticsearch/reference/index.html +# +# ---------------------------------- Cluster ----------------------------------- +# +# Use a descriptive name for your cluster: +# +cluster.name: rally-cluster # Unique name for your cluster +# +# ------------------------------------ Node ------------------------------------ +# +# Use a descriptive name for the node: +# +node.name: node-1 # Unique name for this node +node.roles: [ master, data, ingest ] # Roles assigned to this node +# +# Add custom attributes to the node: +# +#node.attr.rack: r1 +# +# ----------------------------------- Paths ------------------------------------ +# +# Path to directory where to store the data (separate multiple locations by comma): +# +path.data: $.parameters.path.data +# +# Path to log files: +# +path.logs: $.parameters.path.logs +# +# ----------------------------------- Memory ----------------------------------- +# +# Lock the memory on startup: +# +bootstrap.memory_lock: true # Prevent swapping +# +# Make sure that the heap size is set to about half the memory available +# on the system and that the owner of the process is allowed to use this +# limit. +# +# Elasticsearch performs poorly when the system is swapping the memory. +# +# ---------------------------------- Network ----------------------------------- +# +# By default Elasticsearch is only accessible on localhost. Set a different +# address here to expose this node on the network: +# +network.host: 0.0.0.0 # Bind address +# +# By default Elasticsearch listens for HTTP traffic on the first free port it +# finds starting at 9200. Set a specific HTTP port here: +# +http.port: $.parameters.port +# +# For more information, consult the network module documentation. +# +# --------------------------------- Discovery ---------------------------------- +# +# Pass an initial list of hosts to perform discovery when this node is started: +# The default list of hosts is ["127.0.0.1", "[::1]"] +# +#discovery.seed_hosts: ["host1", "host2"] +# +# Bootstrap the cluster using an initial set of master-eligible nodes: +# +#cluster.initial_master_nodes: ["node-1", "node-2"] +discovery.type: single-node +# +# For more information, consult the discovery and cluster formation module documentation. +# +# ---------------------------------- Various ----------------------------------- +# +# Allow wildcard deletion of indices: +# +#action.destructive_requires_name: false + +#----------------------- BEGIN SECURITY AUTO CONFIGURATION ----------------------- +# +# The following settings, TLS certificates, and keys have been automatically +# generated to configure Elasticsearch security features on 25-12-2025 17:58:39 +# +# -------------------------------------------------------------------------------- + +# Enable security features +xpack.security.enabled: false +xpack.security.transport.ssl.enabled: false +xpack.security.http.ssl.enabled: false + + +# Enable encryption for HTTP API client connections, such as Kibana, Logstash, and Agents +# xpack.security.http.ssl: +# enabled: true +# keystore.path: certs/http.p12 + +# Enable encryption and mutual authentication between cluster nodes +# xpack.security.transport.ssl: +# enabled: true +# verification_mode: certificate +# keystore.path: certs/transport.p12 +# truststore.path: certs/transport.p12 +# Create a new cluster with the current node only +# Additional nodes can still join the cluster later +#cluster.initial_master_nodes: ["6be45737fab-1"] + +# Allow HTTP API connections from anywhere +# Connections are encrypted and require user authentication +http.host: 0.0.0.0 + +# Allow other nodes to join the cluster from anywhere +# Connections are encrypted and mutually authenticated +#transport.host: 0.0.0.0 + +#----------------------- END SECURITY AUTO CONFIGURATION ------------------------- \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/limits.ini b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/limits.ini new file mode 100644 index 0000000000..6762991fd7 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Actions/ElasticsearchRally/limits.ini @@ -0,0 +1,76 @@ +# /etc/security/limits.conf +# +#This file sets the resource limits for the users logged in via PAM. +#It does not affect resource limits of the system services. +# +#Also note that configuration files in /etc/security/limits.d directory, +#which are read in alphabetical order, override the settings in this +#file in case the domain is the same or more specific. +#That means, for example, that setting a limit for wildcard domain here +#can be overridden with a wildcard setting in a config file in the +#subdirectory, but a user specific setting here can be overridden only +#with a user specific setting in the subdirectory. +# +#Each line describes a limit for a user in the form: +# +# +# +#Where: +# can be: +# - a user name +# - a group name, with @group syntax +# - the wildcard *, for default entry +# - the wildcard %, can be also used with %group syntax, +# for maxlogin limit +# - NOTE: group and wildcard limits are not applied to root. +# To apply a limit to the root user, must be +# the literal username root. +# +# can have the two values: +# - \"soft\" for enforcing the soft limits +# - \"hard\" for enforcing hard limits +# +# can be one of the following: +# - core - limits the core file size (KB) +# - data - max data size (KB) +# - fsize - maximum filesize (KB) +# - memlock - max locked-in-memory address space (KB) +# - nofile - max number of open file descriptors +# - rss - max resident set size (KB) +# - stack - max stack size (KB) +# - cpu - max CPU time (MIN) +# - nproc - max number of processes +# - as - address space limit (KB) +# - maxlogins - max number of logins for this user +# - maxsyslogins - max number of logins on the system +# - priority - the priority to run user process with +# - locks - max number of file locks the user can hold +# - sigpending - max number of pending signals +# - msgqueue - max memory used by POSIX message queues (bytes) +# - nice - max nice priority allowed to raise to values: [-20, 19] +# - rtprio - max realtime priority +# - chroot - change root to directory (Debian-specific) +# +# +# + +#* soft core 0 +#root hard core 100000 +#* hard rss 10000 +#@student hard nproc 20 +#@faculty soft nproc 20 +#@faculty hard nproc 50 +#ftp hard nproc 0 +#ftp - chroot /ftp +#@student - maxlogins 4 + +* soft memlock unlimited +* hard memlock unlimited +root soft memlock unlimited +root hard memlock unlimited +elasticsearch soft memlock unlimited +elasticsearch hard memlock unlimited +junovmadmin soft memlock unlimited +junovmadmin hard memlock unlimited + +# End of file \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj b/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj index 31975ac868..14da763b75 100644 --- a/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj +++ b/src/VirtualClient/VirtualClient.Actions/VirtualClient.Actions.csproj @@ -22,13 +22,14 @@ - - - - - - + + + + + + + diff --git a/src/VirtualClient/VirtualClient.Main/profiles/PERF-ELASTICSEARCH-RALLY.json b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ELASTICSEARCH-RALLY.json new file mode 100644 index 0000000000..951591a738 --- /dev/null +++ b/src/VirtualClient/VirtualClient.Main/profiles/PERF-ELASTICSEARCH-RALLY.json @@ -0,0 +1,72 @@ +{ + "Description": "Elasticsearch Rally Workload", + "Metadata": { + "RecommendedMinimumExecutionTime": "24:00:00", + "SupportedPlatforms": "linux-x64,linux-arm64", + "SupportedOperatingSystems": "Debian,Ubuntu" + }, + "Parameters": { + "DiskFilter": "osdisk:false&biggestsize", + "ElasticsearchVersion": "9.2.3", + "RallyVersion": "2.12.0", + "Port": "9200" + }, + "Actions": [ + { + "Type": "ElasticsearchRallyServerExecutor", + "Parameters": { + "Scenario": "SetupElasticsearchRallyCluster", + "Port": "$.Parameters.Port", + "ElasticsearchVersion": "$.Parameters.ElasticsearchVersion", + "DiskFilter": "$.Parameters.DiskFilter", + "PackageName": "elasticsearchrally", + "Role": "Server" + } + }, + { + "Type": "ElasticsearchRallyClientExecutor", + "Parameters": { + "MetricScenario": "ESRally_Race_so_vector_index-and-search", + "RallyTrackName": "so_vector", + "RallyCommandLineArguments": "--challenge=index-and-search --include-tasks=index-append,knn-search-10-50-match-all", + "RallyVersion": "$.Parameters.RallyVersion", + "ElasticsearchVersion": "$.Parameters.ElasticsearchVersion", + "Port": "$.Parameters.Port", + "DiskFilter": "$.Parameters.DiskFilter", + "Role": "Client" + } + } + ], + "Dependencies": [ + { + "Type": "FormatDisks", + "Parameters": { + "Scenario": "FormatDisks" + }, + "Role": "Server" + }, + { + "Type": "MountDisks", + "Parameters": { + "Scenario": "CreateMountPoints", + "MountLocation": "mnt" + }, + "Role": "Server" + }, + { + "Type": "LinuxPackageInstallation", + "Parameters": { + "Scenario": "InstallLinuxPackages", + "Packages": "python3,python3-pip,python3-venv,pipx,git,pbzip2", + "Role": "Client" + } + }, + { + "Type": "ApiServer", + "Parameters": { + "Scenario": "StartAPIServer", + "Role": "Server" + } + } + ] +} \ No newline at end of file diff --git a/website/docs/workloads/elasticsearch/elasticsearch-profiles.md b/website/docs/workloads/elasticsearch/elasticsearch-profiles.md new file mode 100644 index 0000000000..41ed7a934d --- /dev/null +++ b/website/docs/workloads/elasticsearch/elasticsearch-profiles.md @@ -0,0 +1,95 @@ +# ElasticSearch Workload Profiles + +The following profiles run customer-representative or benchmarking scenarios using the Rally workload. + +* [Workload Details](./elasticsearch.md) +* [Client/Server Workloads](../../guides/0002-getting-started-client-server.md) + +## Client/Server Topology Support + +Rally (Elasticsearch) workload profiles support running the workload on both a single system as well as in a client/server topology. This means that the workload supports +operation on a single system or on 2 distinct systems. The client/server topology is typically used when it is desirable to include a network component in the +overall performance evaluation. In a client/server topology, one system operates in the 'Client' role making calls to the system operating in the 'Server' role. +The Virtual Client instances running on the client and server systems will synchronize with each other before running the workload. In order to support a client/server topology, +an environment layout file MUST be supplied to each instance of the Virtual Client on the command line to describe the IP address/location of other Virtual Client instances. An +environment layout file is not required for the single system topology. + +* [Environment Layouts](../../guides/0020-client-server.md) + +In the environment layout file provided to the Virtual Client, define the role of the client system/VM as "Client" and the role of the server system(s)/VM(s) as "Server". +The spelling of the roles must be exact. The IP addresses of the systems/VMs must be correct as well. The following example illustrates the +idea. The name of the client must match the name of the system or the value of the agent ID passed in on the command line. + +``` bash +# Single System (environment layout not required) +./VirtualClient --profile=PERF-ELASTICSEARCH-RALLY.json --system=Juno --timeout=1440 + +# Multi-System +# On Client Role System... +./VirtualClient --profile=PERF-ELASTICSEARCH-RALLY.json --system=Juno --timeout=1440 --clientId=ElasticCoordinator --layoutPath=/any/path/to/layout.json + +# On Server Role System... +./VirtualClient --profile=PERF-ELASTICSEARCH-RALLY.json --system=Juno --timeout=1440 --clientId=ElasticServer --layoutPath=/any/path/to/layout.json + +# Example contents of the 'layout.json' file: +{ + "clients": [ + { + "name": "ElasticCoordinator", + "role": "Client", + "ipAddress": "10.1.0.1" + }, + { + "name": "ElasticServer", + "role": "Server", + "ipAddress": "10.1.0.2" + } + ] +} +``` + +## PERF-ELASTICSEARCH-RALLY.json + +Runs a system-intensive workload using the Rally-Elasticsearch benchmark tool. + +* [Workload Profile](https://github.com/microsoft/VirtualClient/blob/main/src/VirtualClient/VirtualClient.Main/profiles/PERF-ELASTICSEARCH-RALLY.json) + +* **Supported Platform/Architectures** + * linux-x64 + * linux-arm64 + +* **Supports Disconnected Scenarios** + * No. Internet connection required. + +* **Dependencies** + The dependencies defined in the 'Dependencies' section of the profile itself are required in order to run the workload operations effectively. + * Internet connection. + + Additional information on components that exist within the 'Dependencies' section of the profile can be found in the following locations: + * [Installing Dependencies](https://microsoft.github.io/VirtualClient/docs/category/dependencies/) + +* **Profile Parameters** + The following parameters can be optionally supplied on the command line. See the 'Usage Scenarios/Examples' above for examples on how to supply parameters to Virtual Client profiles. + + | Parameter | Purpose | Default | + |------------|-------------|--------------| + | DiskFilter | Optional. Configures VC to select a disk for Rally to store benchmark data on. | osdisk:false&biggestsize | + | TrackName | Required. Name of the track for Rally to run. | N/A | + | DistributionVersion | Optional. Version of Elasticsearch to benchmark | 8.0.0 | + | MetricVerbosity | Optional. Sets Metric Verbosity of telemetry. 1=Critical Metrics Only, 2=Standard Metrics, 5=All Metrics | 5 | + +* **Profile Runtimes** + See the 'Metadata' section of the profile for estimated runtimes. These timings represent the length of time required to run a single round of profile actions. These timings can be used to determine minimum required runtimes for the Virtual Client in order to get results. These are often estimates based on the + number of system cores. + +* **Usage Examples** + The following section provides a few basic examples of how to use the workload profile. + + ```bash + # When running on a single system (environment layout not required) + ./VirtualClient --profile=PERF-ELASTICSEARCH-RALLY.json --system=Demo --timeout=1440 --packageStore="{BlobConnectionString|SAS Uri}" + + # When running in a client/server environment + ./VirtualClient --profile=PERF-ELASTICSEARCH-RALLY.json --system=Demo --timeout=1440 --clientId=ElasticCoordinator --layout="/any/path/to/layout.json" --packageStore="{BlobConnectionString|SAS Uri} + ./VirtualClient --profile=PERF-ELASTICSEARCH-RALLY.json --system=Demo --timeout=1440 --clientId=ElasticServer --layout="/any/path/to/layout.json" --packageStore="{BlobConnectionString|SAS Uri}" + ``` diff --git a/website/docs/workloads/elasticsearch/elasticsearch.md b/website/docs/workloads/elasticsearch/elasticsearch.md new file mode 100644 index 0000000000..5337a4e7e2 --- /dev/null +++ b/website/docs/workloads/elasticsearch/elasticsearch.md @@ -0,0 +1,76 @@ +# ElasticSearch Rally + +Rally is an open-source tool that benchmarks Elasticsearch. It can also: + +* Setup and teardown a new Elasticsearch cluster +* Manage benchmark data across Elasticsearch versions +* Runs various benchmarks and collects metrics +* Compare performance results + +This is all possible on single VM, server-client, and cluster-client scenarios. + +* [Rally Source Code](https://github.com/elastic/rally) +* [Rally Documentation](https://esrally.readthedocs.io/en/stable/) + +## Rally Setup + +Rally is only supported on Linux. With the right dependencies installed, Rally can be installed with the following command: + +``` bash +python3 -m pip install esrally +``` + +VC installs esrally in an isolated Python environment using venv to install and run Rally without elevated privileges. + +For best results, it is recommended to alter the root directories of the rally.ini config file (under the /home/user/.rally directory) to point to the data directory of an attached disk. The default is to place this on the OS disk, but the benchmark files may take up more space than is available (nyc_taxis track, for example, takes up 79G). VC automatically takes care of this on execution. + +Furthermore, on the server-side, it is best to set the vm.max_map_count to the maximum value. This expands the amount of memory map areas a process has. Without increasing it, Rally may fail if it does not have enough memory map areas to prepare and operate. + +On a client-server or client-cluster scenario, the Rally daemon should be started as follows, allowing for seamless communication: + +```bash +# on the client +esrallyd start --node-ip=IP_OF_COORDINATOR_NODE --coordinator-ip=IP_OF_COORDINATOR_NODE + +# on any server node +esrallyd start --node-ip=IP_OF_THIS_NODE --coordinator-ip=IP_OF_COORDINATOR_NODE +``` + +Then, rally can be executed with the following command on the client to set up an Elasticsearch node on any servers, and run the benchmark from the coordinator-side. + +```bash +esrally race --track={track_name} --distribution-version={distribution_version} --target-hosts={server_ip_1}:39200,{server_ip_2}:39200 +``` + +If only the benchmark execution is needed, ie. an Elasticsearch server already exists (either on Windows or Linux), Rally can be run with the option --pipeline=benchmark-only. + +## What is Being Measured? + +Rally hosts a various amount of "tracks", each of which are essentially test suites with a collection of tasks. Some tasks are similar between tracks (eg. "index", "index-append", "default"), and some are unique to the track. Tracks can also be created independent of Rally and used by the workload. Examples of popular tracks are listed below. + +| Track Name | Description | Size (in GB) | +|-------------------|---------------|---------------| +| geonames | POIs from Geonames | 3.6 | +| geopoint | Point coordinates from PlanetOSM | 2.8 | +| http_logs | HTTP server log data | 33 | +| nyc_taxis | Taxi rides in New York in 2015 | 79 | +| noaa | Global daily weather measurements from NOAA | 10 | +| pmc | Full text benchmark with academic papers from PMC | 5.5 | + +## Workload Metrics + +The following metrics are examples of those captured by Virtual Client when running the Rally-Elasticsearch workload with the geonames track. Utilize the RallyClientExecutor Parameter `"MetricsFilters": "Verbosity:5"` for metrics of higher detail. + +Each task is measured for throughput, service time, processing time, and latency. Throughput metrics are recorded for median, min, max, and mean. The others are recorded for 50th, 90th, 99th, and 100th percentiles, as well as the mean. With a metrics verbosity of 2 (the VC default), the median throughput as well as the 50th and 90th percentiles are taken to output in the final telemetry. + +| Metric Name | Example Value | Unit | +|-------------|---------------|------| +| index-append_throughput_median | 191.7326520 | docs/s | +| index-append_service_time_50_0 | 3151.838972 | ms | +| index-append_latency_90_0 | 16505.7971 | ms | +| term_throughput_median | 19.8840623 | ops/s | +| term_latency_50_0 | 13.6623771 | ms | +| term_processing_time_90_0 | 16.5516432 | ms | +| scroll_throughput_median | 12.5401983 | pages/s | +| scroll_service_time_90_0 | 950.841824 | ms | +| scroll_processing_time_50_0 | 925.6542731 | ms |