diff --git a/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs b/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs index c1df1a2e73..e428e6bdcc 100644 --- a/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs +++ b/src/VirtualClient/VirtualClient.Actions.UnitTests/3DMark/ThreeDMarkMetricsParserTests.cs @@ -3,62 +3,75 @@ namespace VirtualClient.Actions { + using System; using System.Collections.Generic; - using System.Diagnostics; + using System.ComponentModel; using System.IO; using System.Linq; - using System.Reflection; + using System.Reflection.Metadata; using System.Text; - using System.Threading.Tasks; - using NUnit.Framework; - using VirtualClient.Common; + using System.Text.RegularExpressions; + using System.Xml.Linq; + using System.Xml.Serialization; using VirtualClient.Contracts; - [TestFixture] - [Category("Unit")] - public class ThreeDMarkMetricsParserTests + /// + /// Parser for the NTttcp workload. + /// + public class ThreeDMarkMetricsParser : MetricsParser { - [Test] - public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSGT1() + /// + /// Parser for the 3DMark workload + /// + /// The raw text from the 3DMark export process. + /// The 3dmark definition. + public ThreeDMarkMetricsParser(string rawText, string definition) + : base(rawText) { - string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsgt1.xml"); - string rawText = File.ReadAllText(outputPath); - - ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSGT1.3dmdef"); - IList metrics = testParser.Parse(); - - Assert.AreEqual(1, metrics.Count); - MetricAssert.Exists(metrics, "timespy.graphics.1 [fps]", 59.65, "fps"); + this.Defintion = definition; } - [Test] - public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSGT2() - { - string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsgt2.xml"); - string rawText = File.ReadAllText(outputPath); + private string Defintion { get; set; } - ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSGT2.3dmdef"); - IList metrics = testParser.Parse(); + /// + public override IList Parse() + { + IList metrics = new List(); + try + { + if (this.Defintion == "custom_TSGT1.3dmdef") + { + metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); + } + else if (this.Defintion == "custom_TSGT2.3dmdef") + { + metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); + } + else if (this.Defintion == "custom_TSCT.3dmdef") + { + metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); + } + } + catch (Exception exc) + { + throw new WorkloadException($"Results not found. The workload '3DMark' did not produce any valid results.", exc, ErrorReason.WorkloadFailed); + } - Assert.AreEqual(1, metrics.Count); - MetricAssert.Exists(metrics, "timespy.graphics.2 [fps]", 58.10, "fps"); + return metrics; } - [Test] - public void ThreeDMarkMetricsParserTestsCorrectly_ScenarioTSCT() + /// + /// Gets the value sandwiched between the given XML tags + /// + /// + /// + private double ParseXMLTag(string tagName) { - string workingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location); - string outputPath = Path.Combine(workingDirectory, "Examples", "3DMark", "result_tsct.xml"); - string rawText = File.ReadAllText(outputPath); - - ThreeDMarkMetricsParser testParser = new ThreeDMarkMetricsParser(rawText, "custom_TSCT.3dmdef"); - IList metrics = testParser.Parse(); - - Assert.AreEqual(1, metrics.Count); - MetricAssert.Exists(metrics, "timespy.cpu [fps]", 28.50, "fps"); + string pattern = $"<{tagName}.*>((.|\\n)*?)<\\/{tagName}>"; + Match m = Regex.Match(this.RawText, pattern); + XElement tag = XElement.Parse(m.Value); + double val = double.Parse(tag.Value); + return val; } - } -} \ No newline at end of file +} diff --git a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs index fac39805d9..e428e6bdcc 100644 --- a/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/3DMark/ThreeDMarkMetricsParser.cs @@ -41,15 +41,15 @@ public override IList Parse() { if (this.Defintion == "custom_TSGT1.3dmdef") { - metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("timespy.graphics.1 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest1"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); } else if (this.Defintion == "custom_TSGT2.3dmdef") { - metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("timespy.graphics.2 [fps]", this.ParseXMLTag("TimeSpyPerformanceGraphicsTest2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); } else if (this.Defintion == "custom_TSCT.3dmdef") { - metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric("timespy.cpu [fps]", this.ParseXMLTag("TimeSpyPerformanceCpuSection2"), "fps", MetricRelativity.HigherIsBetter, verbosity: 1)); } } catch (Exception exc) diff --git a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs index 85525a3b56..519b03f2d5 100644 --- a/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/7zip/Compression7zipMetricsParser.cs @@ -62,11 +62,11 @@ public override IList Parse() int rows = this.SizeAndTime.Rows.Count; metrics.Add(new Metric( - "Compression_Ratio", - (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, + "Compression_Ratio", + (Convert.ToDouble(this.SizeAndTime.Rows[2].ItemArray[2]) / Convert.ToDouble(this.SizeAndTime.Rows[0].ItemArray[2])) * 100, "precentage", - relativity: MetricRelativity.LowerIsBetter, - verbosity: 2)); + relativity: MetricRelativity.LowerIsBetter, + verbosity: 5)); double compressionTime = 0; @@ -75,7 +75,7 @@ public override IList Parse() compressionTime += Convert.ToDouble(this.SizeAndTime.Rows[i].ItemArray[2]); } - metrics.Add(new Metric("Compression_Time", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 0)); + metrics.Add(new Metric("Compression_Time", compressionTime, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 1)); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs index 9aa2736eb9..0e99a9e05c 100644 --- a/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/ASPNET/BombardierMetricsParser.cs @@ -32,18 +32,18 @@ public override IList Parse() this.metrics = new List(); Root root = JsonSerializer.Deserialize(this.PreprocessedText); - this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - - this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); - this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency Max", root.Result.Latency.Max, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric("Latency Average", root.Result.Latency.Mean, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + this.metrics.Add(new Metric("Latency Stddev", root.Result.Latency.Stddev, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric("Latency P50", root.Result.Latency.Percentiles.P50, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + this.metrics.Add(new Metric("Latency P75", root.Result.Latency.Percentiles.P75, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency P90", root.Result.Latency.Percentiles.P90, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency P95", root.Result.Latency.Percentiles.P95, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric("Latency P99", root.Result.Latency.Percentiles.P99, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + + this.metrics.Add(new Metric("RequestPerSecond Max", root.Result.Rps.Max, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); + this.metrics.Add(new Metric("RequestPerSecond Average", root.Result.Rps.Mean, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric("RequestPerSecond Stddev", root.Result.Rps.Stddev, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); this.metrics.Add(new Metric("RequestPerSecond P50", root.Result.Rps.Percentiles.P50, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); this.metrics.Add(new Metric("RequestPerSecond P75", root.Result.Rps.Percentiles.P75, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); this.metrics.Add(new Metric("RequestPerSecond P90", root.Result.Rps.Percentiles.P90, MetricUnit.RequestsPerSec, MetricRelativity.HigherIsBetter)); diff --git a/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs index 6ad353945d..cdc9107e5b 100644 --- a/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/CoreMark/CoreMarkMetricsParser.cs @@ -46,23 +46,23 @@ public override IList Parse() metrics.AddRange(this.CoreMarkResult.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "NA", namePrefix: string.Empty, ignoreFormatError: true)); // CoreMark result doesn't define the unit so needs manually assign units. metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Unit = "bytes"; - metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "CoreMark Size").FirstOrDefault().Verbosity = 5; metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Unit = "ticks"; - metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Total ticks").FirstOrDefault().Verbosity = 5; metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Unit = "secs"; - metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Total time (secs)").FirstOrDefault().Verbosity = 5; metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Unit = "iterations/sec"; metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Relativity = MetricRelativity.HigherIsBetter; - metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Verbosity = 0; + metrics.Where(m => m.Name == "Iterations/Sec").FirstOrDefault().Verbosity = 1; metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Unit = "iterations"; metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Relativity = MetricRelativity.Undefined; - metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Iterations").FirstOrDefault().Verbosity = 5; // This line won't be there if it's running single thread. if (metrics.Any(m => m.Name == "Parallel PThreads")) { metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Unit = "threads"; - metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Verbosity = 2; + metrics.Where(m => m.Name == "Parallel PThreads").FirstOrDefault().Verbosity = 5; } return metrics; diff --git a/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs index a67abe6426..9368d28571 100644 --- a/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/DiskSpd/DiskSpdMetricsParser.cs @@ -182,9 +182,9 @@ private void ParseCPUResult() DataTable cpuUsage = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); - this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[1].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 2)); - this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[2].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 2)); - this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[3].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 2)); + this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[1].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 5)); + this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[2].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 5)); + this.metrics.AddRange(cpuUsage.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "percentage", namePrefix: $"cpu {cpuUsage.Columns[3].ColumnName.ToLower()} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 5)); } private void ParseTotalIoResult() @@ -193,15 +193,16 @@ private void ParseTotalIoResult() DataTable totalIo = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); List metrics = new List(); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"total {totalIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"total {totalIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"total {totalIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"total {totalIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"total {totalIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"total {totalIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"total {totalIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"total {totalIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"total {totalIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(totalIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"total {totalIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); + // Total metrics ending with "total" remain at level 1 (most important) foreach (var metric in metrics.Where(m => m.Name.EndsWith("total") && (m.Unit == "iops" || m.Unit == "ms" || m.Unit == "MiB/s"))) { - metric.Verbosity = 0; + metric.Verbosity = 1; // Keep at 1 for critical } this.metrics.AddRange(metrics); @@ -213,15 +214,15 @@ private void ParseReadIoResult() DataTable readIo = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); List metrics = new List(); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"read {readIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"read {readIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"read {readIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"read {readIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"read {readIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"read {readIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"read {readIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"read {readIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"read {readIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(readIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"read {readIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); foreach (var metric in metrics.Where(m => m.Name.EndsWith("total") && (m.Unit == "iops" || m.Unit == "ms" || m.Unit == "MiB/s"))) { - metric.Verbosity = 0; + metric.Verbosity = 1; } this.metrics.AddRange(metrics); @@ -233,14 +234,15 @@ private void ParseWriteIoResult() DataTable writeIo = DataTableExtensions.ConvertToDataTable( this.Sections[sectionName], DiskSpdMetricsParser.DiskSpdDataTableDelimiter, sectionName, columnNames: null); List metrics = new List(); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"write {writeIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"write {writeIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 2)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"write {writeIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"write {writeIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"write {writeIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "bytes", namePrefix: $"write {writeIo.Columns[1].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "I/Os", namePrefix: $"write {writeIo.Columns[2].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 5)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "MiB/s", namePrefix: $"write {writeIo.Columns[3].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 4, unit: "iops", namePrefix: $"write {writeIo.Columns[4].ColumnName} ", metricRelativity: MetricRelativity.HigherIsBetter, metricVerbosity: 1)); + metrics.AddRange(writeIo.GetMetrics(nameIndex: 0, valueIndex: 5, unit: "ms", namePrefix: $"write {writeIo.Columns[5].ColumnName} ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); + foreach (var metric in metrics.Where(m => m.Name.EndsWith("total") && (m.Unit == "iops" || m.Unit == "ms" || m.Unit == "MiB/s"))) { - metric.Verbosity = 0; + metric.Verbosity = 1; } this.metrics.AddRange(metrics); @@ -254,20 +256,20 @@ private void ParseLatencyResult() List metrics = new List(); if (this.readWriteMode != ReadWriteMode.WriteOnly) { - metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "ms", namePrefix: "read latency ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "ms", namePrefix: "read latency ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); } if (this.readWriteMode != ReadWriteMode.ReadOnly) { - metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "ms", namePrefix: "write latency ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 2, unit: "ms", namePrefix: "write latency ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); } - metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "ms", namePrefix: "total latency ", metricRelativity: MetricRelativity.LowerIsBetter)); + metrics.AddRange(latency.GetMetrics(nameIndex: 0, valueIndex: 3, unit: "ms", namePrefix: "total latency ", metricRelativity: MetricRelativity.LowerIsBetter, metricVerbosity: 1)); string[] criticalMetrics = { "total latency 50th", "total latency 90th", "total latency 99th" }; foreach (var metric in metrics.Where(m => criticalMetrics.Contains(m.Name))) { - metric.Verbosity = 0; + metric.Verbosity = 1; // Keep at 1 for critical percentiles } this.metrics.AddRange(metrics); diff --git a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs index 9e11dbd5fa..07946c2189 100644 --- a/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/FIO/FioMetricsParser.cs @@ -122,29 +122,30 @@ private void AddLatencyHistogramMeasurements(IList metrics, JToken resul { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"latency_us.2", "h000000_002", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.4", "h000000_004", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.10", "h000000_010", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.20", "h000000_020", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.50", "h000000_050", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.100", "h000000_100", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.250", "h000000_250", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.500", "h000000_500", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.750", "h000000_750", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_us.1000", "h000001_000", verbosity: 2); - - this.AddMeasurement(metrics, job, $"latency_ms.2", "h000002_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.4", "h000004_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.10", "h000010_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.20", "h000020_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.50", "h000050_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.100", "h000100_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.250", "h000250_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.500", "h000500_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.750", "h000750_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.1000", "h001000_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.2000", "h002000_000", verbosity: 2); - this.AddMeasurement(metrics, job, $"latency_ms.['>=2000']", "hgt002000_000", verbosity: 2); + // Level 5 - Verbose: Histogram buckets (diagnostic metrics) + this.AddMeasurement(metrics, job, $"latency_us.2", "h000000_002", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.4", "h000000_004", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.10", "h000000_010", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.20", "h000000_020", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.50", "h000000_050", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.100", "h000000_100", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.250", "h000000_250", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.500", "h000000_500", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.750", "h000000_750", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_us.1000", "h000001_000", verbosity: 5); + + this.AddMeasurement(metrics, job, $"latency_ms.2", "h000002_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.4", "h000004_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.10", "h000010_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.20", "h000020_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.50", "h000050_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.100", "h000100_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.250", "h000250_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.500", "h000500_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.750", "h000750_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.1000", "h001000_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.2000", "h002000_000", verbosity: 5); + this.AddMeasurement(metrics, job, $"latency_ms.['>=2000']", "hgt002000_000", verbosity: 5); } } } @@ -162,44 +163,52 @@ private void AddReadMeasurements(IList metrics, JToken resultsJson) { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"read.io_bytes", "read_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"read.total_ios", "read_ios", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"read.short_ios", "read_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"read.drop_ios", "read_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.bw", "read_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_min", "read_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_max", "read_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.bw_mean", "read_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.bw_dev", "read_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.iops", "read_iops", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_min", "read_iops_min", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_max", "read_iops_max", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"read.iops_mean", "read_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.iops_stddev", "read_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.lat_ns.min", "read_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.lat_ns.max", "read_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.lat_ns.mean", "read_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.lat_ns.stddev", "read_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.clat_ns.min", "read_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.max", "read_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.mean", "read_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.stddev", "read_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['50.000000']", "read_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['70.000000']", "read_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['90.000000']", "read_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.000000']", "read_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.990000']", "read_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - - this.AddMeasurement(metrics, job, $"read.slat_ns.min", "read_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.slat_ns.max", "read_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.slat_ns.mean", "read_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"read.slat_ns.stddev", "read_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - + // Level 5 - Verbose/Diagnostic metrics + this.AddMeasurement(metrics, job, $"read.io_bytes", "read_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"read.total_ios", "read_ios", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"read.short_ios", "read_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"read.drop_ios", "read_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical bandwidth metrics (most important) + this.AddMeasurement(metrics, job, $"read.bw", "read_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_min", "read_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_max", "read_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_mean", "read_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.bw_dev", "read_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical IOPS metrics + this.AddMeasurement(metrics, job, $"read.iops", "read_iops", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_min", "read_iops_min", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_max", "read_iops_max", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_mean", "read_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.iops_stddev", "read_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard latency metrics + this.AddMeasurement(metrics, job, $"read.lat_ns.min", "read_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.lat_ns.max", "read_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.lat_ns.mean", "read_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.lat_ns.stddev", "read_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Standard completion latency metrics + this.AddMeasurement(metrics, job, $"read.clat_ns.min", "read_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.max", "read_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.mean", "read_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.stddev", "read_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Critical percentiles + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['50.000000']", "read_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.000000']", "read_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + + // Level 3 - Detailed: Other percentiles + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['70.000000']", "read_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['90.000000']", "read_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"read.clat_ns.percentile.['99.990000']", "read_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + + // Level 1 - Standard submission latency metrics + this.AddMeasurement(metrics, job, $"read.slat_ns.min", "read_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.slat_ns.max", "read_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.slat_ns.mean", "read_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"read.slat_ns.stddev", "read_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); } } } @@ -217,44 +226,52 @@ private void AddWriteMeasurements(IList metrics, JToken resultsJson) { foreach (JToken job in jobs) { - this.AddMeasurement(metrics, job, $"write.io_bytes", "write_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"write.total_ios", "write_ios", null, MetricRelativity.HigherIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"write.short_ios", "write_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 2); - this.AddMeasurement(metrics, job, $"write.drop_ios", "write_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.bw", "write_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_min", "write_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_max", "write_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.bw_mean", "write_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.bw_dev", "write_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.iops", "write_iops", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_min", "write_iops_min", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_max", "write_iops_max", null, MetricRelativity.HigherIsBetter); - this.AddMeasurement(metrics, job, $"write.iops_mean", "write_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.iops_stddev", "write_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.lat_ns.min", "write_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.lat_ns.max", "write_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.lat_ns.mean", "write_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.lat_ns.stddev", "write_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.clat_ns.min", "write_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.max", "write_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.mean", "write_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.stddev", "write_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['50.000000']", "write_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['70.000000']", "write_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['90.000000']", "write_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.000000']", "write_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 0); - this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.990000']", "write_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - - this.AddMeasurement(metrics, job, $"write.slat_ns.min", "write_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.slat_ns.max", "write_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.slat_ns.mean", "write_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor); - this.AddMeasurement(metrics, job, $"write.slat_ns.stddev", "write_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); - + // Level 5 - Verbose/Diagnostic metrics + this.AddMeasurement(metrics, job, $"write.io_bytes", "write_bytes", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"write.total_ios", "write_ios", null, MetricRelativity.HigherIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"write.short_ios", "write_ios_short", null, MetricRelativity.LowerIsBetter, verbosity: 5); + this.AddMeasurement(metrics, job, $"write.drop_ios", "write_ios_dropped", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical bandwidth metrics + this.AddMeasurement(metrics, job, $"write.bw", "write_bandwidth", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_min", "write_bandwidth_min", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_max", "write_bandwidth_max", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_mean", "write_bandwidth_mean", MetricUnit.KibibytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.bw_dev", "write_bandwidth_stdev", MetricUnit.KibibytesPerSecond, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard/Critical IOPS metrics + this.AddMeasurement(metrics, job, $"write.iops", "write_iops", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_min", "write_iops_min", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_max", "write_iops_max", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_mean", "write_iops_mean", null, MetricRelativity.HigherIsBetter, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.iops_stddev", "write_iops_stdev", null, MetricRelativity.LowerIsBetter, verbosity: 5); + + // Level 1 - Standard latency metrics + this.AddMeasurement(metrics, job, $"write.lat_ns.min", "write_latency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.lat_ns.max", "write_latency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.lat_ns.mean", "write_latency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.lat_ns.stddev", "write_latency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Standard completion latency metrics + this.AddMeasurement(metrics, job, $"write.clat_ns.min", "write_completionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.max", "write_completionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.mean", "write_completionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.stddev", "write_completionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); + + // Level 1 - Critical percentiles + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['50.000000']", "write_completionlatency_p50", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.000000']", "write_completionlatency_p99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + + // Level 3 - Detailed: Other percentiles + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['70.000000']", "write_completionlatency_p70", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['90.000000']", "write_completionlatency_p90", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + this.AddMeasurement(metrics, job, $"write.clat_ns.percentile.['99.990000']", "write_completionlatency_p99_99", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 2); + + // Level 1 - Standard submission latency metrics + this.AddMeasurement(metrics, job, $"write.slat_ns.min", "write_submissionlatency_min", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.slat_ns.max", "write_submissionlatency_max", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.slat_ns.mean", "write_submissionlatency_mean", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 1); + this.AddMeasurement(metrics, job, $"write.slat_ns.stddev", "write_submissionlatency_stdev", this.ConversionUnits, MetricRelativity.LowerIsBetter, this.ConversionFactor, verbosity: 5); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs index 740cb33262..5a8dcff803 100644 --- a/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/GeekBench/GeekBenchMetricsParser.cs @@ -79,7 +79,7 @@ public override IList Parse() metrics.AddRange(this.SingleCoreSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: "SingleCoreSummary-", metricRelativity: MetricRelativity.HigherIsBetter)); metrics.AddRange(this.MultiCoreSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: "MultiCoreSummary-", metricRelativity: MetricRelativity.HigherIsBetter)); - metrics.ForEach(metric => metric.Verbosity = 0); + metrics.ForEach(metric => metric.Verbosity = 1); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs index b0b2a8e583..242080260b 100644 --- a/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/HPLinpack/HPLinpackMetricsParser.cs @@ -81,8 +81,8 @@ public override IList Parse() { $"Q_W{st[0]}", st[4] }, }; - this.metrics.Add(new Metric($"Time", Convert.ToDouble(st[5]), "secs", MetricRelativity.Undefined, metadata: metadata, verbosity: 2)); - this.metrics.Add(new Metric($"GFlops", Convert.ToDouble(st[6]), "Gflops", metadata: metadata, relativity: MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric($"Time", Convert.ToDouble(st[5]), "secs", MetricRelativity.Undefined, metadata: metadata, verbosity: 5)); + this.metrics.Add(new Metric($"GFlops", Convert.ToDouble(st[6]), "Gflops", metadata: metadata, relativity: MetricRelativity.HigherIsBetter, verbosity: 1)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs index 48a17b8eef..a973c73062 100644 --- a/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/HammerDB/HammerDBMetricsParser.cs @@ -58,14 +58,14 @@ public override IList Parse() metrics.First(m => m.Name == "Operations/min").Value / 60, MetricUnit.OperationsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0)); + verbosity: 1)); metrics.Add(new Metric( "Transactions/sec", metrics.First(m => m.Name == "Transactions/min").Value / 60, MetricUnit.TransactionsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0)); + verbosity: 1)); return metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs index 687c3237ee..ae3d1980de 100644 --- a/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Memtier/MemtierMetricsParser.cs @@ -25,7 +25,7 @@ public class MemtierMetricsParser : MetricsParser // Sets 4827.17 --- --- 2.64323 2.83100 3.93500 4.47900 7.45500 29.56700 329.45 // Waits 0.00 --- --- --- --- --- --- --- --- --- // Totals 48271.29 43444.12 0.00 2.62213 2.75100 3.90300 4.41500 7.42300 29.31100 4053.46 - + private static readonly Regex BandwidthExpression = new Regex(@"Bandwidth|Throughput", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex GetsExpression = new Regex(@"(?<=Gets).*(?=\n)", RegexOptions.Compiled | RegexOptions.IgnoreCase); private static readonly Regex GetLatencyP80Expression = new Regex(@"(?<=GET)\s+([0-9\.]+)\s+.*80[\.0]+", RegexOptions.Compiled | RegexOptions.IgnoreCase); @@ -308,93 +308,103 @@ private static IDictionary CreateMetricsBin(string metr { return new Dictionary { - // e.g. - // Throughput-Req/sec - // GET_Throughput-Req/sec - // GET_Throughput-Req/sec - // Latency-Avg - // Latency-Avg (Gets) - // Latency-Avg (Sets) + // Level 1 - Critical: Throughput { "Ops/sec", new MetricAggregate( metricNamePrefix == null ? "Throughput" : $"{metricNamePrefix}-Throughput", metricUnit: MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0, + verbosity: 1, // Level 1 - Most important description: "Total number of requests/operations per second during the period of time.") }, + // Level 1 - Standard: Hits/Misses { "Hits/sec", new MetricAggregate( metricNamePrefix == null ? "Hits/sec" : $"{metricNamePrefix}-Hits/sec", + metricUnit: MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, + verbosity: 1, description: "Total number of cache hits per second during the period of time.") }, { "Misses/sec", new MetricAggregate( metricNamePrefix == null ? "Misses/sec" : $"{metricNamePrefix}-Misses/sec", + metricUnit: MetricUnit.RequestsPerSec, relativity: MetricRelativity.LowerIsBetter, - description: "Total number of cache misses per second during a period of time. This is an indication of data evictions due to reaching memory limits.") + verbosity: 1, + description: "Total number of cache misses per second during a period of time.") }, + // Level 1 - Standard: Average latency { "AvgLatency", new MetricAggregate( metricNamePrefix == null ? "Latency-Avg" : $"{metricNamePrefix}-Latency-Avg", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 1, description: "Average latency for requests/operations during the period of time.") }, + // Level 1 - Critical: P50 latency { "p50Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P50" : $"{metricNamePrefix}-Latency-P50", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 0, + verbosity: 1, description: "The latency for 50% of all requests was at or under this value.") }, + // Level 3 - Detailed: P90 latency { "p90Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P90" : $"{metricNamePrefix}-Latency-P90", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 2, description: "The latency for 90% of all requests was at or under this value.") }, + // Level 3 - Detailed: P95 latency { "p95Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P95" : $"{metricNamePrefix}-Latency-P95", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 2, description: "The latency for 95% of all requests was at or under this value.") }, + // Level 1 - Critical: P99 latency { "p99Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P99" : $"{metricNamePrefix}-Latency-P99", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 0, + verbosity: 1, description: "The latency for 99% of all requests was at or under this value.") }, + // Level 3 - Detailed: P99.9 latency { "p99.9Latency", new MetricAggregate( metricNamePrefix == null ? "Latency-P99.9" : $"{metricNamePrefix}-Latency-P99.9", metricUnit: MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, + verbosity: 2, description: "The latency for 99.9% of all requests was at or under this value.") }, + // Level 1 - Critical: Bandwidth { "KB/sec", new MetricAggregate( metricNamePrefix == null ? "Bandwidth" : $"{metricNamePrefix}-Bandwidth", metricUnit: MetricUnit.KilobytesPerSecond, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0, + verbosity: 1, description: "Total amount of data transferred per second during the period of time.") } }; diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs index 1d1489957b..5cc60cd807 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/CPS/CPSMetricsParser.cs @@ -82,24 +82,24 @@ private static void AddStatisticalMetrics(IList metrics, List ti double lowerCI = mean - t; double upperCI = mean + t; - metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min())); - metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max())); - metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median())); - metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25))); - metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50))); - metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75))); - metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90))); - metrics.Add(new Metric("ConnectsPerSec_P99", connectsPerSec.Percentile(99))); - metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3))); - metrics.Add(new Metric("ConnectsPerSec_P99_99", Statistics.QuantileCustom(connectsPerSec, 1d - 0.0001d, QuantileDefinition.R3))); - metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3))); + metrics.Add(new Metric("ConnectsPerSec_Min", connectsPerSec.Min(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_Max", connectsPerSec.Max(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_Med", connectsPerSec.Median(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_Avg", connectsPerSec.Average(), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric("ConnectsPerSec_P25", connectsPerSec.Percentile(25), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_P50", connectsPerSec.Percentile(50), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric("ConnectsPerSec_P75", connectsPerSec.Percentile(75), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_P90", connectsPerSec.Percentile(90), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_P99", connectsPerSec.Percentile(99), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric("ConnectsPerSec_P99_9", Statistics.QuantileCustom(connectsPerSec, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_P99_99", Statistics.QuantileCustom(connectsPerSec, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_P99_999", Statistics.QuantileCustom(connectsPerSec, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.TransactionsPerSec, MetricRelativity.HigherIsBetter, verbosity: 5)); double median = Statistics.Median(connectsPerSec); double[] absoluteDeviations = connectsPerSec.Select(x => Math.Abs(x - median)).ToArray(); - metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations), MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("ConnectsPerSec_Mad", Statistics.Median(absoluteDeviations), MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_StandardErrorMean", sem, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_LowerCI", lowerCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("ConnectsPerSec_UpperCI", upperCI, MetricUnit.TransactionsPerSec, MetricRelativity.LowerIsBetter, verbosity: 5)); } /// @@ -109,7 +109,7 @@ private static KeyValuePair, List> GetTimestampsAndConnecti { bool appendResult = true; int valueIndex = 0; - + // The timestamps and connectionsPerSecond co-relate on the index. List timestamps = new List(); List connectionsPerSec = new List(); diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs index c09ac0905a..36b4772caa 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/Latte/LatteMetricsParser.cs @@ -144,7 +144,7 @@ private void AddLatencyPercentileMetrics(IList metrics, IEnumerable entry.FrequencyInTotal <= totalObservations * .75) @@ -178,7 +178,7 @@ private void AddLatencyPercentileMetrics(IList metrics, IEnumerable entry.FrequencyInTotal <= totalObservations * .999) diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs index b0f0d7c913..ca9c9ccd56 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/NTttcp/NTttcpMetricsParser.cs @@ -19,7 +19,7 @@ public class NTttcpMetricsParser : MetricsParser { private readonly bool isClient; private NTttcpResult result; - + /// /// Parser for the NTttcp workload /// @@ -51,14 +51,14 @@ public override IList Parse() NTttcpMetric throughputMetric = this.result.Throughput.First(t => t.Units.Equals("mbps")); IList metricList = new List() { - new Metric("TotalBytesMB", this.result.TotalBytesMB.Value, this.result.TotalBytesMB.Units, MetricRelativity.HigherIsBetter, verbosity: 2), - new Metric("AvgBytesPerCompl", this.result.AverageBytesPerCompletion.Value, this.result.AverageBytesPerCompletion.Units, MetricRelativity.Undefined, verbosity: 2), - new Metric("AvgFrameSize", this.result.AverageFrameSize.Value, this.result.AverageFrameSize.Units, MetricRelativity.Undefined, verbosity: 2), - new Metric("ThroughputMbps", throughputMetric.Value, throughputMetric.Units, MetricRelativity.HigherIsBetter, verbosity: 0), - new Metric("AvgPacketsPerInterrupt", this.result.AveragePacketsPerInterrupt.Value, this.result.AveragePacketsPerInterrupt.Units, MetricRelativity.Undefined, verbosity: 2), - new Metric("InterruptsPerSec", this.result.Interrupts.Value, this.result.Interrupts.Units, MetricRelativity.Undefined), - new Metric("PacketsRetransmitted", this.result.PacketsRetransmitted, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 2), - new Metric("Errors", this.result.Errors, MetricUnit.Count, MetricRelativity.LowerIsBetter), + new Metric("TotalBytesMB", this.result.TotalBytesMB.Value, this.result.TotalBytesMB.Units, MetricRelativity.HigherIsBetter, verbosity: 5), + new Metric("AvgBytesPerCompl", this.result.AverageBytesPerCompletion.Value, this.result.AverageBytesPerCompletion.Units, MetricRelativity.Undefined, verbosity: 5), + new Metric("AvgFrameSize", this.result.AverageFrameSize.Value, this.result.AverageFrameSize.Units, MetricRelativity.Undefined, verbosity: 5), + new Metric("ThroughputMbps", throughputMetric.Value, throughputMetric.Units, MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("AvgPacketsPerInterrupt", this.result.AveragePacketsPerInterrupt.Value, this.result.AveragePacketsPerInterrupt.Units, MetricRelativity.Undefined, verbosity: 5), + new Metric("InterruptsPerSec", this.result.Interrupts.Value, this.result.Interrupts.Units, MetricRelativity.Undefined, verbosity: 2), + new Metric("PacketsRetransmitted", this.result.PacketsRetransmitted, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 5), + new Metric("Errors", this.result.Errors, MetricUnit.Count, MetricRelativity.LowerIsBetter, verbosity: 2), }; if (this.result.TcpAverageRtt != null) @@ -133,7 +133,7 @@ public override IList Parse() /// do not use outside the context of /// public class NTttcpResult - { + { /// /// A collection of key value pairs passed into the NTttcp workload. /// diff --git a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs index 248f29e5d9..63d0d0e8a0 100644 --- a/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Network/NetworkingWorkload/SockPerf/SockPerfMetricsParser.cs @@ -107,24 +107,24 @@ private static void AddStatisticalMetrics(IList metrics, List pa double lowerCI = mean - t; double upperCI = mean + t; - metrics.Add(new Metric("Latency-Min", packetsLatencyValues.Min(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-Max", packetsLatencyValues.Max(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-Avg", packetsLatencyValues.Average(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P25", packetsLatencyValues.Percentile(25), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P50", packetsLatencyValues.Percentile(50), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - metrics.Add(new Metric("Latency-P75", packetsLatencyValues.Percentile(75), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P90", packetsLatencyValues.Percentile(90), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P99", packetsLatencyValues.Percentile(99), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 0)); - metrics.Add(new Metric("Latency-P99.9", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P99.99", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); - metrics.Add(new Metric("Latency-P99.999", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric("Latency-Min", packetsLatencyValues.Min(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-Max", packetsLatencyValues.Max(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-Avg", packetsLatencyValues.Average(), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + metrics.Add(new Metric("Latency-P25", packetsLatencyValues.Percentile(25), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-P50", packetsLatencyValues.Percentile(50), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + metrics.Add(new Metric("Latency-P75", packetsLatencyValues.Percentile(75), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-P90", packetsLatencyValues.Percentile(90), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-P99", packetsLatencyValues.Percentile(99), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 1)); + metrics.Add(new Metric("Latency-P99.9", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-P99.99", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.0001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-P99.999", Statistics.QuantileCustom(packetsLatencyValues, 1d - 0.00001d, QuantileDefinition.R3), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); double median = Statistics.Median(packetsLatencyValues); double[] absoluteDeviations = packetsLatencyValues.Select(x => Math.Abs(x - median)).ToArray(); - metrics.Add(new Metric("Latency-Mad", Statistics.Median(absoluteDeviations), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-StandardErrorMean", sem, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-Stdev", sd, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-LowerCI", lowerCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); - metrics.Add(new Metric("Latency-UpperCI", upperCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 2)); + metrics.Add(new Metric("Latency-Mad", Statistics.Median(absoluteDeviations), MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-StandardErrorMean", sem, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-Stdev", sd, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-LowerCI", lowerCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); + metrics.Add(new Metric("Latency-UpperCI", upperCI, MetricUnit.Microseconds, MetricRelativity.LowerIsBetter, verbosity: 5)); } } } \ No newline at end of file diff --git a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs index c1a348b6a9..a33780bbd8 100644 --- a/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/OpenSSL/OpenSslMetricsParser.cs @@ -171,7 +171,7 @@ public override IList Parse() // aes-128-cbc -1040172814.93 1589805.17k 1657786.91k 1674053.70k 1677365.52k 1633415.99k if (metricValue >= 0) { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.KilobytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } } } @@ -187,11 +187,11 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 1)); } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 2)); } } } @@ -208,11 +208,11 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 1)); } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 2)); } } } @@ -229,11 +229,11 @@ public override IList Parse() { if (metricName.Contains("/")) { - metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(metricName, metricValue, $"{row[OpenSslMetricsParser.ColumnUnit]}", MetricRelativity.HigherIsBetter, verbosity: 1)); } else { - metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter)); + metrics.Add(new Metric(metricName, metricValue, MetricUnit.Seconds, MetricRelativity.LowerIsBetter, verbosity: 2)); } } } diff --git a/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs index 779a7ccd05..a5c68fdcc8 100644 --- a/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/OpenSSL/TlsOpenSslMetricsParser.cs @@ -61,20 +61,20 @@ public override IList Parse() // add metrics to the list - // set 1 - new connection metrics - metrics.Add(new Metric(TotalBytesRead, parsedMetrics[TotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(NoOfConnections, parsedMetrics[NoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(Duration, parsedMetrics[Duration], MetricUnit.Seconds)); - metrics.Add(new Metric(BytesperConnection, parsedMetrics[BytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(NewConnThroughput, newConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(NewConnPersec, newConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(TotalBytesRead, parsedMetrics[TotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(NoOfConnections, parsedMetrics[NoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(Duration, parsedMetrics[Duration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 2)); + metrics.Add(new Metric(BytesperConnection, parsedMetrics[BytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(NewConnThroughput, newConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(NewConnPersec, newConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); // set 2 - reuse connection metrics - metrics.Add(new Metric(ReuseTotalBytesRead, parsedMetrics[ReuseTotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseNoOfConnections, parsedMetrics[ReuseNoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseDuration, parsedMetrics[ReuseDuration], MetricUnit.Seconds)); - metrics.Add(new Metric(ReuseBytesperConnection, parsedMetrics[ReuseBytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseConnThroughput, reuseConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - metrics.Add(new Metric(ReuseConnPerSec, reuseConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 0)); + metrics.Add(new Metric(ReuseTotalBytesRead, parsedMetrics[ReuseTotalBytesRead], MetricUnit.Bytes, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseNoOfConnections, parsedMetrics[ReuseNoOfConnections], MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseDuration, parsedMetrics[ReuseDuration], MetricUnit.Seconds, MetricRelativity.Undefined, verbosity: 2)); + metrics.Add(new Metric(ReuseBytesperConnection, parsedMetrics[ReuseBytesperConnection], MetricUnit.BytesPerConnection, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseConnThroughput, reuseConnThroughput, MetricUnit.BytesPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + metrics.Add(new Metric(ReuseConnPerSec, reuseConnPerSec, MetricUnit.Count, MetricRelativity.HigherIsBetter, verbosity: 1)); return metrics; } @@ -101,10 +101,10 @@ Now timing with session id reuse. 1696 connections in 0.86s; 1972.09 connections/user sec, bytes read 547808 1696 connections in 6 real seconds, 323 bytes read per connection */ - + // ignore the first connections and connections per user metric as it is conflicting with // total connections in n seconds - + // Initial run var match1 = Regex.Match(input, @"(\d+) connections in ([\d.]+)s; ([\d.]+) connections/user sec, bytes read (\d+)"); if (match1.Success) diff --git a/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs index 5c7904bd3e..52d55a75f8 100644 --- a/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/Redis/RedisBenchmarkMetricsParser.cs @@ -108,7 +108,7 @@ private void AddMetricsFromCsv(List metrics) reqPerSec, MetricUnit.RequestsPerSec, relativity: MetricRelativity.HigherIsBetter, - verbosity: 0, + verbosity: 1, description: "Total number of requests/operations per second during the period of time.")); } @@ -159,7 +159,7 @@ private void AddMetricsFromCsv(List metrics) p99Latency, MetricUnit.Milliseconds, relativity: MetricRelativity.LowerIsBetter, - verbosity: 0, + verbosity: 1, description: "The latency for 99% of all requests was at or under this value.")); } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs index bb045d4ca8..fe75f85113 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECcpu/SpecCpuMetricsParser.cs @@ -68,7 +68,7 @@ public override IList Parse() this.metrics.AddRange(this.SpecCpuSummary.GetMetrics(nameIndex: 0, valueIndex: 1, unit: "Score", namePrefix: string.Empty, ignoreFormatError: true, metricRelativity: MetricRelativity.HigherIsBetter)); // Every score in SPECcpu is critical metric. - this.metrics.ForEach(m => m.Verbosity = 0); + this.metrics.ForEach(m => m.Verbosity = 1); return this.metrics; } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs index 9d671963f6..bec610a828 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECjbb/SpecJbbMetricsParser.cs @@ -49,7 +49,7 @@ public override IList Parse() } else { - this.metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric(name.Trim(), Convert.ToDouble(value), SpecJbbMetricsParser.OperationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs index f546bfaffd..4b0401fcf7 100644 --- a/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/SPECjvm/SpecJvmMetricsParser.cs @@ -46,7 +46,7 @@ public override IList Parse() string[] nameAndValue = Regex.Split(line, columnRegex.ToString(), columnRegex.Options); string metricName = nameAndValue[0].Trim(); double metricValue = Convert.ToDouble(nameAndValue[1].Replace(SpecJvmMetricsParser.operationPerSecond, string.Empty)); - this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } else { @@ -55,7 +55,7 @@ public override IList Parse() string metricName = nameAndValue[0].Trim(); string[] a = line.Split(" "); double metricValue = Convert.ToDouble(nameAndValue[1]); - this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); + this.metrics.Add(new Metric(metricName, metricValue, SpecJvmMetricsParser.operationPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); } } diff --git a/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs b/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs index 8f8a008a2f..6e04c1df51 100644 --- a/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs +++ b/src/VirtualClient/VirtualClient.Actions/StressNg/StressNgMetricsParser.cs @@ -42,12 +42,12 @@ public override IList Parse() foreach (StressNgStressorResult stressor in parsedResult.Metrics) { - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops", stressor.BogoOps, "BogoOps", MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-usr-sys-time", stressor.BogoOpsPerSecondUsrSysTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-real-time", stressor.BogoOpsPerSecondRealTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 0)); - this.metrics.Add(new Metric($"{stressor.Stressor}-wall-clock-time", stressor.WallClockTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric($"{stressor.Stressor}-user-time", stressor.UserTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); - this.metrics.Add(new Metric($"{stressor.Stressor}-system-time", stressor.SystemTime, "second", MetricRelativity.LowerIsBetter, verbosity: 2)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops", stressor.BogoOps, "BogoOps", MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-usr-sys-time", stressor.BogoOpsPerSecondUsrSysTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric($"{stressor.Stressor}-bogo-ops-per-second-real-time", stressor.BogoOpsPerSecondRealTime, BogusOperationsPerSecond, MetricRelativity.HigherIsBetter, verbosity: 1)); + this.metrics.Add(new Metric($"{stressor.Stressor}-wall-clock-time", stressor.WallClockTime, "second", MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric($"{stressor.Stressor}-user-time", stressor.UserTime, "second", MetricRelativity.LowerIsBetter, verbosity: 5)); + this.metrics.Add(new Metric($"{stressor.Stressor}-system-time", stressor.SystemTime, "second", MetricRelativity.LowerIsBetter, verbosity: 5)); } return this.metrics; diff --git a/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs b/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs index f16a0c3fa2..fdea920376 100644 --- a/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs +++ b/src/VirtualClient/VirtualClient.Contracts.UnitTests/MetricExtensionsTests.cs @@ -38,11 +38,11 @@ public void SetupTest() new Metric("write_completionlatency_p99_99", 3267543), new Metric("write_submissionlatency_mean", 15.35467863), - new Metric("verbose_test_1", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 0), - new Metric("verbose_test_2", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 0), + new Metric("verbose_test_1", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("verbose_test_2", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), new Metric("verbose_test_3", 123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), new Metric("verbose_test_4", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), - new Metric("verbose_test_5", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), + new Metric("verbose_test_5", -123, "unit", MetricRelativity.HigherIsBetter, verbosity: 5), }; } @@ -71,12 +71,12 @@ public void MetricFiltersCorrectFiltersVerbosity() IEnumerable filter = new List { "Verbosity:1" }; CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 1).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); - filter = new List { "Verbosity:0" }; - CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity == 0).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); - filter = new List { "Verbosity:2" }; CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 2).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); + filter = new List { "Verbosity:5" }; + CollectionAssert.AreEquivalent(this.metrics.Where(m => m.Verbosity <= 5).Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); + filter = new List { "Verbosity:others" }; CollectionAssert.AreEquivalent(Enumerable.Empty().Select(m => m.Name), this.metrics.FilterBy(filter).Select(m => m.Name)); } @@ -172,7 +172,7 @@ public void FilterByExtensionReturnsTheExpectedFilteredWithBothVerbosityAndText( List filters = new List { "test_2", - "verbosity:0", + "verbosity:1", }; IEnumerable expectedMetrics = this.metrics.Where(m => m.Name == "verbose_test_2"); @@ -182,5 +182,142 @@ public void FilterByExtensionReturnsTheExpectedFilteredWithBothVerbosityAndText( Assert.IsNotEmpty(actualMetrics); CollectionAssert.AreEquivalent(expectedMetrics, actualMetrics); } + + [Test] + public void FilterByExtensionSupportsAllVerbosityLevels() + { + // Setup metrics with different verbosity levels (1, 2, 5 only) + var metrics = new List + { + new Metric("critical_metric_1a", 1, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("critical_metric_1b", 2, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("detailed_metric_2a", 3, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), + new Metric("detailed_metric_2b", 4, "unit", MetricRelativity.HigherIsBetter, verbosity: 2), + new Metric("verbose_metric_5a", 5, "unit", MetricRelativity.HigherIsBetter, verbosity: 5), + new Metric("verbose_metric_5b", 6, "unit", MetricRelativity.HigherIsBetter, verbosity: 5) + }; + + // Test verbosity:1 - should return only level 1 metrics + var filter1 = new List { "verbosity:1" }; + var result1 = metrics.FilterBy(filter1); + Assert.AreEqual(2, result1.Count()); + Assert.IsTrue(result1.All(m => m.Verbosity == 1)); + + // Test verbosity:2 - should return level 1 + level 2 metrics + var filter2 = new List { "verbosity:2" }; + var result2 = metrics.FilterBy(filter2); + Assert.AreEqual(4, result2.Count()); + Assert.IsTrue(result2.All(m => m.Verbosity <= 2)); + + // Test verbosity:5 - should return all metrics + var filter5 = new List { "verbosity:5" }; + var result5 = metrics.FilterBy(filter5); + Assert.AreEqual(6, result5.Count()); + } + + [Test] + public void FilterByExtensionHandlesInvalidVerbosityValues() + { + var metrics = new List + { + new Metric("test_metric", 1, "unit", MetricRelativity.HigherIsBetter, verbosity: 1) + }; + + // Test invalid verbosity format + var invalidFilter1 = new List { "verbosity:invalid" }; + var result1 = metrics.FilterBy(invalidFilter1); + Assert.AreEqual(0, result1.Count()); + + // Test out of range verbosity (> 5) + var invalidFilter2 = new List { "verbosity:10" }; + var result2 = metrics.FilterBy(invalidFilter2); + Assert.AreEqual(0, result2.Count()); + + // Test verbosity less than 0 (negative) + var invalidFilter3 = new List { "verbosity:-1" }; + var result3 = metrics.FilterBy(invalidFilter3); + Assert.AreEqual(0, result3.Count()); + } + + [Test] + public void FilterByExtensionHandlesBackwardCompatibilityForVerbosityZero() + { + var metrics = new List + { + new Metric("critical_metric", 1, "unit", MetricRelativity.HigherIsBetter, verbosity: 0), + new Metric("standard_metric", 2, "unit", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("detailed_metric", 3, "unit", MetricRelativity.HigherIsBetter, verbosity: 2) + }; + + // verbosity:0 should be mapped to verbosity:1 for backward compatibility + var filter = new List { "verbosity:0" }; + var result = metrics.FilterBy(filter); + + // Should include metrics with verbosity 0 and 1 (since 0 maps to 1) + Assert.AreEqual(2, result.Count()); + Assert.IsTrue(result.Any(m => m.Name == "critical_metric")); + Assert.IsTrue(result.Any(m => m.Name == "standard_metric")); + Assert.IsFalse(result.Any(m => m.Name == "detailed_metric")); + } + + [Test] + public void FilterByExtensionSupportsExclusionFilters() + { + var metrics = new List + { + new Metric("h000_metric", 1), + new Metric("h001_metric", 2), + new Metric("bandwidth", 3), + new Metric("iops", 4) + }; + + // Test exclusion filter + var exclusionFilter = new List { "-h000*", "-h001*" }; + var result = metrics.FilterBy(exclusionFilter); + Assert.AreEqual(2, result.Count()); + Assert.IsFalse(result.Any(m => m.Name.StartsWith("h00"))); + Assert.IsTrue(result.Any(m => m.Name == "bandwidth")); + Assert.IsTrue(result.Any(m => m.Name == "iops")); + } + + [Test] + public void FilterByExtensionComposesVerbosityAndNameFilters() + { + var metrics = new List + { + new Metric("bandwidth_read", 1, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("bandwidth_write", 2, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("iops_read", 3, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("iops_write", 4, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("latency_p99", 5, "ms", MetricRelativity.LowerIsBetter, verbosity: 5) + }; + + // Filter: verbosity <= 1 AND name contains "bandwidth" + var composedFilter = new List { "verbosity:1", "bandwidth" }; + var result = metrics.FilterBy(composedFilter); + Assert.AreEqual(2, result.Count()); + Assert.IsTrue(result.All(m => m.Name.Contains("bandwidth") && m.Verbosity <= 1)); + } + + [Test] + public void FilterByExtensionSupportsComplexFilterCombinations() + { + var metrics = new List + { + new Metric("h000_latency", 1, "ms", MetricRelativity.LowerIsBetter, verbosity: 5), + new Metric("h001_latency", 2, "ms", MetricRelativity.LowerIsBetter, verbosity: 5), + new Metric("bandwidth_read", 3, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("bandwidth_write", 4, "MB/s", MetricRelativity.HigherIsBetter, verbosity: 1), + new Metric("iops", 5, "ops/s", MetricRelativity.HigherIsBetter, verbosity: 1) + }; + + // Complex filter: verbosity <= 5, exclude h00* metrics, include only bandwidth or iops + var complexFilter = new List { "verbosity:5", "-h00*", "bandwidth|iops" }; + var result = metrics.FilterBy(complexFilter); + + Assert.IsTrue(result.All(m => m.Verbosity <= 5)); + Assert.IsFalse(result.Any(m => m.Name.StartsWith("h00"))); + Assert.IsTrue(result.All(m => m.Name.Contains("bandwidth") || m.Name.Contains("iops"))); + } } } diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs b/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs index c9b2c2475d..2ef9547e7d 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensibility/MetricDataPoint.cs @@ -77,12 +77,16 @@ public MetricDataPoint(TelemetryDataPoint dataPoint) public double? MetricValue; /// - /// The priority/verbosity of the metric. Recommended Values = 0 (Critical), 1 (Standard), 2 (Informational) etc.. + /// The priority/verbosity of the metric. Recommended Values: + /// - 1 (Standard/Critical): Most important metrics + /// - 3 (Detailed): Additional detailed metrics + /// - 5 (Verbose): All diagnostic/internal metrics /// /// /// Allows the user to ascribe different levels of priority/verbosity to a set of metrics that can /// be used for queries/filtering. Lower values indicate higher priority. For example, metrics considered - /// to be the most critical for decision making would be set with verbosity = 0 (Critical). + /// to be the most critical for decision making would be set with verbosity = 1 (Critical). + /// For backward compatibility, verbosity = 0 is mapped to verbosity = 1. /// [JsonProperty("metricVerbosity", Required = Required.Default)] [YamlMember(Alias = "metricVerbosity", ScalarStyle = ScalarStyle.Plain)] diff --git a/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs b/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs index 402022c4b5..fe9a23cb67 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Extensions/MetricExtensions.cs @@ -33,14 +33,32 @@ public static void AddMetadata(this IEnumerable metrics, IDictionary - /// Filters the set of metrics down to those whose names match or contain the filter terms - /// provided (case-insensitive). + /// Filters the set of metrics down to those that match the filter criteria provided. + /// Supports both verbosity-based filtering and regex-based name filtering (case-insensitive). /// /// The set of metrics to filter down. - /// A set of terms to match against the metric names. + /// + /// A set of filter terms. Can include: + /// - Verbosity filters: "verbosity:N" where N is 1-5 (filters metrics with verbosity less than or equal to N) + /// - Name filters: Any regex pattern to match against metric names (case-insensitive) + /// - Exclusion filters: Prefix with "-" to exclude matching metrics (e.g., "-h000*") + /// /// - /// A set of metrics whose names match or contain the filter terms (case-insensitive). + /// A filtered set of metrics matching the criteria. /// + /// + /// Verbosity levels define a convention for organizing metrics by importance: + /// - 1 (Standard/Critical): Most important metrics - bandwidth, throughput, IOPS, key latency percentiles (p50, p99) + /// - 2 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 3 (Reserved): Reserved for future expansion + /// - 4 (Reserved): Reserved for future expansion + /// - 5 (Verbose): All diagnostic/internal metrics - histogram buckets, standard deviations, byte counts, I/O counts + /// + /// Currently, only levels 1, 2, and 5 are actively used. Levels 3 and 4 are reserved for future use. + /// + /// For backward compatibility, verbosity:0 is mapped to verbosity:1. + /// Filters are composable - verbosity filtering is applied first, then name-based filtering. + /// public static IEnumerable FilterBy(this IEnumerable metrics, IEnumerable filterTerms) { metrics.ThrowIfNull(nameof(metrics)); @@ -49,32 +67,45 @@ public static IEnumerable FilterBy(this IEnumerable metrics, IEn if (filterTerms?.Any() == true) { - string verbosityFilter = filterTerms.Where(f => f.Contains("verbosity", StringComparison.OrdinalIgnoreCase)).FirstOrDefault(); - if (!string.IsNullOrEmpty(verbosityFilter)) + // Step 1: Handle verbosity filtering first + string verbosityFilter = filterTerms.FirstOrDefault(f => f.Contains("verbosity", StringComparison.OrdinalIgnoreCase)); + if (!string.IsNullOrEmpty(verbosityFilter)) { - switch (verbosityFilter.ToLower()) + // Extract the verbosity level from the filter (e.g., "verbosity:3" -> 3) + string[] parts = verbosityFilter.Split(':'); + if (parts.Length == 2 && int.TryParse(parts[1].Trim(), out int maxVerbosity) && maxVerbosity >= 0 && maxVerbosity <= 5) { - case "verbosity:0": - filteredMetrics = filteredMetrics.Where(m => ((int)m.Verbosity) <= 0); - break; - case "verbosity:1": - filteredMetrics = filteredMetrics.Where(m => ((int)m.Verbosity) <= 1); - break; - case "verbosity:2": - filteredMetrics = filteredMetrics.Where(m => ((int)m.Verbosity) <= 2); - break; - - default: - filteredMetrics = Enumerable.Empty(); - break; + // Backward compatibility: Map old level 0 to new level 1 + if (maxVerbosity == 0) + { + maxVerbosity = 1; + } + + // Filter metrics to include only those with verbosity <= maxVerbosity + filteredMetrics = filteredMetrics.Where(m => m.Verbosity <= maxVerbosity); + } + else + { + // Invalid verbosity format or out of range - return empty set + filteredMetrics = Enumerable.Empty(); } + // Remove verbosity filter from remaining filters filterTerms = filterTerms.Where(f => !f.Contains("verbosity", StringComparison.OrdinalIgnoreCase)); } + // Step 2: Handle exclusion filters (prefix with "-") + var exclusionFilters = filterTerms.Where(f => f.StartsWith("-")).Select(f => f.Substring(1)).ToList(); + if (exclusionFilters.Any()) + { + filteredMetrics = filteredMetrics.Where(m => !exclusionFilters.Contains(m.Name, MetricFilterComparer.Instance)); + filterTerms = filterTerms.Where(f => !f.StartsWith("-")); + } + + // Step 3: Handle inclusion/regex name filtering if (filterTerms?.Any() == true) { - filteredMetrics = metrics.Where(m => filterTerms.Contains(m.Name, MetricFilterComparer.Instance)).ToList(); + filteredMetrics = filteredMetrics.Where(m => filterTerms.Contains(m.Name, MetricFilterComparer.Instance)); } } @@ -102,7 +133,8 @@ public static void LogConsole(this IEnumerable metrics, string scenario, table.Columns.Add("Value", typeof(double)); table.Columns.Add("Unit", typeof(string)); - IEnumerable metricsToPrint = criticalOnly ? metrics.Where(m => m.Verbosity == 0).ToList() : metrics; + // Support both old (0) and new (1) critical levels for backward compatibility + IEnumerable metricsToPrint = criticalOnly ? metrics.Where(m => m.Verbosity == 0 || m.Verbosity == 1).ToList() : metrics; foreach (Metric metric in metricsToPrint) { diff --git a/src/VirtualClient/VirtualClient.Contracts/Metric.cs b/src/VirtualClient/VirtualClient.Contracts/Metric.cs index fa152ef9d6..2bd7bee971 100644 --- a/src/VirtualClient/VirtualClient.Contracts/Metric.cs +++ b/src/VirtualClient/VirtualClient.Contracts/Metric.cs @@ -26,7 +26,7 @@ public Metric(string name, double value) this.Name = name; this.Value = value; this.Relativity = MetricRelativity.Undefined; - this.Metadata = new Dictionary(); + this.Metadata = new Dictionary(); this.Tags = new List(); } @@ -140,8 +140,18 @@ public Metric(string name, double value, string unit, MetricRelativity relativit public IDictionary Metadata { get; } /// - /// Metric verbosity to descript importance of metric. Default to 1, which means standard. - /// Verbosity 0: Critical. Verbosity 1: Standard. Verbosity 2: Informational. + /// Metric verbosity to describe importance/priority of the metric. + /// + /// Verbosity levels define a convention for organizing metrics by importance: + /// - 1 (Standard/Critical): Most important metrics for decision making - bandwidth, throughput, IOPS, key latency percentiles (p50, p99) + /// - 2 (Detailed): Additional detailed metrics - supplementary percentiles (p70, p90, p95, p99.9) + /// - 3 (Reserved): Reserved for future expansion + /// - 4 (Reserved): Reserved for future expansion + /// - 5 (Verbose): All diagnostic/internal metrics - histogram buckets, standard deviations, byte counts, I/O counts + /// + /// Currently, only levels 1, 2, and 5 are actively used. Levels 3 and 4 are reserved for future use. + /// + /// Default = 1 (Standard). /// public int Verbosity { get; set; } = 1; diff --git a/website/docs/guides/0011-profiles.md b/website/docs/guides/0011-profiles.md index 2aaa0b7e7c..11c7b3aa5f 100644 --- a/website/docs/guides/0011-profiles.md +++ b/website/docs/guides/0011-profiles.md @@ -382,6 +382,104 @@ e.g. ] ``` +## Metric Filtering +The Virtual Client supports filtering of metrics emitted by workloads and monitors using the `MetricFilters` parameter. This allows users to control which metrics are captured and +logged, reducing telemetry volume and focusing on metrics of interest. Metric filtering supports three filtering strategies: **verbosity-based filtering**, **regex-based inclusion +filtering**, and **exclusion filtering**. + +### Verbosity-Based Filtering +Metrics in the Virtual Client are assigned a verbosity level that indicates their importance for decision making: + +| Verbosity Level | Description | Example Metrics | +|-----------------|-------------|-----------------| +| 1 (Standard/Critical) | Most important metrics for decision making | bandwidth, throughput, IOPS, p50, p99 | +| 2 (Reserved) | Reserved for future expansion | N/A | +| 3 (Detailed) | Additional detailed metrics | p70, p90, p95, p99.9 | +| 4 (Reserved) | Reserved for future expansion | N/A | +| 5 (Verbose) | All diagnostic/internal metrics | histogram buckets, standard deviations, byte counts, I/O counts | + +To filter metrics by verbosity level, use the `verbosity:N` filter format, where N is a value between 1 and 5. This will include all metrics with a verbosity level less than or equal to N. + +``` json +"Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "PackageName": "fio", + "CommandLine": "--name=fio_test --size=10G --rw=randwrite --bs=4k", + "MetricFilters": "verbosity:1" + } + } +] +``` + +### Regex-Based Inclusion Filtering +Metrics can be filtered using regular expression patterns (case-insensitive). Multiple filter terms can be combined, and a metric will be included if it matches any of the patterns. + +``` json +"Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "PackageName": "fio", + "CommandLine": "--name=fio_test --size=10G --rw=randwrite --bs=4k", + "MetricFilters": "bandwidth,iops,_p99" + } + } +] +``` + +The filter above will include metrics whose names contain "bandwidth", "iops", or "_p99". More complex regex patterns are also supported: + +``` json +"MetricFilters": "(read|write)_(bandwidth|iops)" +``` + +### Exclusion Filtering +Metrics can be explicitly excluded by prefixing the filter term with a minus sign (`-`). This is useful for removing verbose or diagnostic metrics that are not needed. + +``` json +"Actions": [ + { + "Type": "RedisServerExecutor", + "Parameters": { + "Scenario": "Server", + "PackageName": "redis", + "CommandLine": "--protected-mode no", + "MetricFilters": "-h000*,-h001*" + } + } +] +``` + +### Combining Filter Types +Multiple filter types can be combined. Verbosity filtering is applied first, followed by exclusion filters, and then inclusion filters. + +``` json +"Actions": [ + { + "Type": "FioExecutor", + "Parameters": { + "Scenario": "RandomWrite_4k_BlockSize", + "PackageName": "fio", + "CommandLine": "--name=fio_test --size=10G --rw=randwrite --bs=4k", + "MetricFilters": "verbosity:3,-histogram*,bandwidth|iops|latency" + } + } +] +``` + +The example above will: +1. Include only metrics with verbosity ? 3 +2. Exclude metrics matching the pattern "histogram*" +3. From the remaining metrics, include only those matching "bandwidth", "iops", or "latency" + +### Default Behavior +When the `MetricFilters` parameter is not specified, all metrics emitted by the workload or monitor are captured and logged. This ensures backward compatibility with existing +profiles and allows users to see all available metrics during initial testing before applying filters to reduce telemetry volume. + ## Actions The section 'Actions' within the profile defines a set of 1 or more toolsets to execute on the system. The term 'workload' is often used synonymously with 'Actions' to describe the software that is the focus of the work Virtual Client will perform on the system. For example, the section might include an action responsible for executing