diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index 892d9446b4..2a9abe9fd4 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -15,6 +15,46 @@ There are several categories of characteristics which you can specify. Let's con It's a single string characteristic. It allows to name your job. This name will be used in logs and a part of a folder name with generated files for this job. `Id` doesn't affect benchmark results, but it can be useful for diagnostics. If you don't specify `Id`, random value will be chosen based on other characteristics +### Categories + +Job categories are the job equivalent of `[BenchmarkCategory]`: they allow you to group jobs so that you can select which ones to run. A job can belong to any number of categories, and the comparison is case insensitive. + +```cs +var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithCategories("runtimes", "net8")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core90).WithCategories("runtimes", "net9")) + .AddJob(Job.Dry.WithCategory("debug")); +``` + +`WithCategories` overrides the categories of the job, `WithCategory` adds to them. + +Every job attribute has a `Categories` property, so the same thing can be expressed with attributes: + +```cs +[SimpleJob(RuntimeMoniker.Net80, Categories = ["runtimes", "net8"])] +[SimpleJob(RuntimeMoniker.Net90, Categories = ["runtimes", "net9"])] +public class Benchmarks { /* ... */ } +``` + +Categories don't affect how a job is executed: they are not a part of the job `Id`, the folder names, nor the summary. + +To run only the jobs which belong to given categories, use `JobCategoryFilter`, `[JobCategoryFilter]` or the `--jobCategories` console argument: + +```cs +var config = DefaultConfig.Instance.AddFilter(new JobCategoryFilter(["net8"])); +``` + +```cs +[JobCategoryFilter("net8")] +public class Benchmarks { /* ... */ } +``` + +```log +dotnet run -c Release -- --jobCategories net8 +``` + +Like `--anyCategories` and `--allCategories`, the filter is inclusion only: a job with no categories belongs to none of the requested ones, so it is filtered out. If your config mixes categorized and uncategorized jobs, selecting a category also removes the uncategorized jobs, so give a category to every job you may want to keep. + ### Environment `Environment` specifies an environment of the job. You can specify the following characteristics: diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index eecfb42208..920b0f6c87 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -103,6 +103,7 @@ You can also filter the benchmarks by categories: * `--anyCategories` - runs all benchmarks that belong to **any** of the provided categories * `--allCategories`- runs all benchmarks that belong to **all** provided categories +* `--jobCategories` - runs all benchmarks whose job belongs to **any** of the provided job categories. Jobs without categories are excluded ## Diagnosers @@ -272,6 +273,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `--allStats` (Default: false) Displays all statistics (min, max & more) * `--allCategories` Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed * `--anyCategories` Any Categories to run +* `--jobCategories` Job categories to run. Only the benchmarks which belong to a job that has any of them are going to be executed * `--attribute` Run all methods with given attribute (applied to class or method) * `--join` (Default: false) Prints single table with results for all benchmarks * `--title` Custom title for the produced summaries and the base name of the result files exported for them diff --git a/src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs new file mode 100644 index 0000000000..1b55c36151 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Filters; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// Runs only the benchmarks whose job belongs to any of the given job categories, + /// see . + /// + public class JobCategoryFilterAttribute : FilterConfigBaseAttribute + { + // CLS-Compliant Code requires a constructor without an array in the argument list + [PublicAPI] + public JobCategoryFilterAttribute() { } + + public JobCategoryFilterAttribute(params string[] targetCategories) : base(new JobCategoryFilter(targetCategories)) { } + } +} diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs index 0ece453a6e..94e8f78dbf 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs @@ -9,13 +9,34 @@ namespace BenchmarkDotNet.Attributes [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] public class JobConfigBaseAttribute : Attribute, IConfigSource { + private readonly Job? job; + private IConfig? config; + // CLS-Compliant Code requires a constructor which use only CLS-compliant types [PublicAPI] - public JobConfigBaseAttribute() => Config = ManualConfig.CreateEmpty(); + public JobConfigBaseAttribute() { } + + protected JobConfigBaseAttribute(Job job) => this.job = job; - protected JobConfigBaseAttribute(Job job) => Config = ManualConfig.CreateEmpty().AddJob(job); + /// + /// The categories of the job defined by this attribute. Categories are metadata used to select which jobs are + /// executed (see ), they don't affect how the job is executed. + /// + /// + /// It is init-only because is built once and then kept: a later assignment would be + /// silently ignored. Attribute named arguments are allowed to set init-only properties. + /// + [PublicAPI] public string[]? Categories { get; init; } - public IConfig Config { get; } + // Named attribute properties are assigned after the constructor has run, so the config cannot be built there: + // Categories would still be empty. It is read long after the attribute is constructed, so building it lazily + // is enough to see the categories the user has set. + public IConfig Config => config ??= job == null + ? ManualConfig.CreateEmpty() + : ManualConfig.CreateEmpty().AddJob( + // `Categories = null` is what an attribute argument of an array type is allowed to be, so it must not + // be dereferenced without a check. + Categories is not { Length: > 0 } categories ? job : job.WithCategories(categories).Freeze()); protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? jit, Platform? platform) { diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 18c1b65fd6..7750f77ea4 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -223,7 +223,10 @@ private static ImmutableHashSet GetValidators(IEnumerable private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) { - var unique = jobs.Distinct(JobComparer.Default).ToArray(); + // JobComparer ignores the categories, so jobs that differ only by category collapse into one here. Their + // categories are merged into the survivor, otherwise selecting by the categories of the collapsed jobs + // would silently match nothing. + var unique = jobs.GroupBy(job => job, JobComparer.Default).Select(MergeCategories).ToArray(); var result = new List(); foreach (var standardJob in unique.Where(job => !job.Meta.IsMutator && !job.Meta.IsDefault)) @@ -241,8 +244,16 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) { var copy = result[i].UnfreezeCopy(); + // Apply overwrites the characteristics, but the categories are additive: a mutator that carries + // categories (eg [SimpleJob(Categories = ...)] on a method) adds them to every job it mutates + // instead of replacing the ones that job already has. Dropping them would make selecting by the + // categories of the mutated job silently match nothing. + var ownCategories = copy.Meta.Categories; + copy.Apply(mutatorJob); + copy.Meta.Categories = [.. ownCategories, .. copy.Meta.Categories]; + result[i] = copy.Freeze(); } } @@ -250,6 +261,19 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) return result; } + /// + /// Returns the first job of a group of jobs which are equal except for their categories, + /// carrying the categories of all of them. + /// + private static Job MergeCategories(IGrouping equalJobs) + { + var first = equalJobs.Key; + var merged = equalJobs.SelectMany(job => job.Meta.Categories).ToArray(); + + // The common case is a group of one, where there is nothing to merge and the job can be returned as it is. + return merged.Length == first.Meta.Categories.Count ? first : first.WithCategories(merged).Freeze(); + } + private class TypeComparer : IEqualityComparer { // different types can implement the same interface, we want to distinct by type diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index cc06b11216..1e07cd3685 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -75,6 +75,9 @@ public bool UseDisassemblyDiagnoser [Option("anyCategories", Required = false, HelpText = "Any Categories to run")] public IEnumerable AnyCategories { get; set; } = []; + [Option("jobCategories", Required = false, HelpText = "Job categories to run. Only the benchmarks which belong to a job that has any of them are going to be executed")] + public IEnumerable JobCategories { get; set; } = []; + [Option("attribute", Required = false, HelpText = "Run all methods with given attribute (applied to class or method)")] public IEnumerable AttributeNames { get; set; } = []; @@ -240,7 +243,7 @@ public bool UseDisassemblyDiagnoser [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")] public bool Resume { get; set; } - internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any(); + internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any() || JobCategories.Any(); [Usage(ApplicationAlias = "")] [PublicAPI] diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 3c15744439..e787cc6d79 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -797,6 +797,8 @@ private static IEnumerable GetFilters(CommandLineOptions options) yield return new AllCategoriesFilter(options.AllCategories.ToArray()); if (options.AnyCategories.Any()) yield return new AnyCategoriesFilter(options.AnyCategories.ToArray()); + if (options.JobCategories.Any()) + yield return new JobCategoryFilter(options.JobCategories.ToArray()); if (options.AttributeNames.Any()) yield return new AttributesFilter(options.AttributeNames.ToArray()); } diff --git a/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs b/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs new file mode 100644 index 0000000000..587d2e2026 --- /dev/null +++ b/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs @@ -0,0 +1,21 @@ +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Filters +{ + /// + /// Filter benchmarks which belong to a job that has any of the target job categories. + /// + /// Like every other category filter, this one is inclusion only: a benchmark whose job has no categories at all + /// belongs to none of the target categories, so it is filtered out. In a config that mixes categorized and + /// uncategorized jobs, selecting a category therefore also removes the uncategorized jobs. + /// + /// + public class JobCategoryFilter : IFilter + { + private readonly string[] targetCategories; + + public JobCategoryFilter(string[] targetCategories) => this.targetCategories = targetCategories; + + public bool Predicate(BenchmarkCase benchmarkCase) => targetCategories.Any(category => benchmarkCase.Job.Meta.HasCategory(category)); + } +} diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 3a46793219..60bb8a1339 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -48,6 +48,12 @@ public int Compare(Job? x, Job? y) foreach (var characteristic in x.GetAllCharacteristics()) { + // Categories say which jobs the user wants to select, they are not a part of what a job *is*. + // Comparing them would let two jobs that differ only by category survive the deduplication done by + // ImmutableConfigBuilder, which would run the same job twice under two identical names. + if (characteristic == MetaMode.CategoriesCharacteristic) + continue; + if (!x.HasValue(characteristic)) { if (y.HasValue(characteristic)) diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 2b5e3ff116..8a83aee1c1 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -324,6 +324,24 @@ public static Job WithMsBuildArguments(this Job job, params string[] msBuildArgu /// public static Job AsDefault(this Job job, bool value = true) => job.WithCore(j => j.Meta.IsDefault = value); + /// + /// Creates a new job based on the given job with the given category added to . + /// categories are metadata used to select which jobs are executed, they don't affect how the job is executed + /// + /// The original job + /// The category which should be added to the new job + /// The new job with the additional category + public static Job WithCategory(this Job job, string category) => job.WithCore(j => j.Meta.AddCategories([category])); + + /// + /// Creates a new job based on the given job with the given categories. + /// the categories of the original job are not preserved, use if you want to add to them + /// + /// The original job + /// The categories of the new job + /// The new job with overriden categories + public static Job WithCategories(this Job job, params string[] categories) => job.WithCore(j => j.Meta.Categories = categories); + internal static Job MakeSettingsUserFriendly(this Job job, Descriptor descriptor) { // users expect that if IterationSetup is configured, it should be run before every benchmark invocation https://github.com/dotnet/BenchmarkDotNet/issues/730 diff --git a/src/BenchmarkDotNet/Jobs/MetaMode.cs b/src/BenchmarkDotNet/Jobs/MetaMode.cs index d57ebe0298..f5e75897ba 100644 --- a/src/BenchmarkDotNet/Jobs/MetaMode.cs +++ b/src/BenchmarkDotNet/Jobs/MetaMode.cs @@ -9,6 +9,19 @@ public class MetaMode : JobMode [PublicAPI] public static readonly Characteristic IsMutatorCharacteristic = CreateIgnoreOnApplyCharacteristic(nameof(IsMutator)); [PublicAPI] public static readonly Characteristic IsDefaultCharacteristic = CreateHiddenCharacteristic(nameof(IsDefault)); + /// + /// the categories of the job, the job equivalent of + /// + /// the characteristic is hidden on purpose: categories are metadata used to select jobs, they must not affect + /// the generated job id (), the folder names, the summary + /// nor the code generated for the child process. + /// it is not ignored on apply, even though categories are additive rather than overwritten: UnfreezeCopy is + /// built on Apply, so an ignored characteristic would be dropped by every WithXxx call. The places that have + /// to add rather than overwrite merge the categories explicitly, see + /// + /// + [PublicAPI] public static readonly Characteristic> CategoriesCharacteristic = CreateHiddenCharacteristic>(nameof(Categories)); + public bool Baseline { get => BaselineCharacteristic[this]; @@ -32,5 +45,56 @@ public bool IsDefault get => IsDefaultCharacteristic[this]; set => IsDefaultCharacteristic[this] = value; } + + /// + /// the categories of the job. Setting it overrides the categories that the job already has, + /// use if you want to add to them. + /// + public IReadOnlyList Categories + { + get => CategoriesCharacteristic[this] ?? []; + + // Assigning null removes the characteristic (see CharacteristicObject.SetValueCore), which is what an + // empty set of categories has to do: a job that was given no categories must stay indistinguishable from + // one that was never given any, otherwise `WithCategories(selection)` on an empty selection would mark + // the job as changed. + set => CategoriesCharacteristic[this] = Unique(value) is { Count: > 0 } unique ? unique : null!; + } + + /// + /// Adds the specified to . + /// The categories that are already present are not duplicated (the comparison is case insensitive). + /// + public void AddCategories(IEnumerable categories) + { + ArgumentNullException.ThrowIfNull(categories); + + Categories = [.. Categories, .. categories]; + } + + /// + /// checks whether the job belongs to given category (the comparison is case insensitive) + /// + public bool HasCategory(string category) => Categories.Contains(category, StringComparer.OrdinalIgnoreCase); + + // Every category assigned to a job goes through here, which is why the arguments are validated here rather + // than in the WithCategory/WithCategories extensions: the property and AddCategories are public too, and + // guarding only the extensions would leave `job.Meta.Categories = null` reported as a null `source` thrown + // out of Distinct, and would let a null category be stored and only fail much later. + private static IReadOnlyList Unique(IEnumerable categories) + { + ArgumentNullException.ThrowIfNull(categories); + + var all = categories.ToArray(); + + // A null category can never be selected and it survives the merging done when jobs are deduplicated, + // so it would surface far away from the call that introduced it. + if (all.Any(category => category is null)) + throw new ArgumentException("A job category must not be null.", nameof(categories)); + + // the categories are used to select jobs, so we don't want the users to end up with the same category + // twice just because they have used a different casing + return [.. all.Distinct(StringComparer.OrdinalIgnoreCase)]; + } } -} \ No newline at end of file +} diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs new file mode 100644 index 0000000000..926775aa88 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs @@ -0,0 +1,392 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Tests.Jobs +{ + public class JobCategoryTests + { + [Fact] + public void JobsHaveNoCategoriesByDefault() + { + Assert.Empty(Job.Default.Meta.Categories); + Assert.False(Job.Default.Meta.HasCategory("anything")); + } + + [Fact] + public void WithCategoriesOverridesTheExistingCategories() + { + var job = Job.Dry.WithCategories("first").WithCategories("second", "third"); + + Assert.Equal(["second", "third"], job.Meta.Categories); + } + + [Fact] + public void WithCategoryAddsToTheExistingCategories() + { + var job = Job.Dry.WithCategories("first").WithCategory("second"); + + Assert.Equal(["first", "second"], job.Meta.Categories); + } + + [Fact] + public void TheSameCategoryIsNotAddedTwice() + { + var job = Job.Dry.WithCategories("dupe", "DUPE").WithCategory("Dupe"); + + Assert.Equal(["dupe"], job.Meta.Categories); + } + + [Theory] + [InlineData("net8")] + [InlineData("NET8")] + [InlineData("Net8")] + public void HasCategoryIsCaseInsensitive(string category) + { + Assert.True(Job.Dry.WithCategory("net8").Meta.HasCategory(category)); + } + + [Fact] + public void HasCategoryIsAnExactMatch() + { + var job = Job.Dry.WithCategory("net8"); + + Assert.False(job.Meta.HasCategory("net")); + Assert.False(job.Meta.HasCategory("net80")); + } + + [Fact] + public void TheOriginalJobIsNotModified() + { + var original = Job.Dry.WithCategory("first"); + + original.WithCategory("second"); + + Assert.Equal(["first"], original.Meta.Categories); + } + + // categories are metadata, they must not change the identity of the job: + // the id is used for the folder names and the summary, and users don't expect it to change + // just because they have categorized their jobs. See MetaMode.CategoriesCharacteristic + [Fact] + public void CategoriesDoNotAffectTheJobId() + { + var job = Job.Default.WithLaunchCount(3); + var categorized = job.WithCategories("net8", "slow"); + + Assert.Equal(job.ResolvedId, categorized.ResolvedId); + Assert.Equal(job.FolderInfo, categorized.FolderInfo); + Assert.Equal(job.DisplayInfo, categorized.DisplayInfo); + } + + [Fact] + public void CategoriesDoNotAffectTheGeneratedJobIdOfTheDefaultJob() + { + Assert.Equal("DefaultJob", Job.Default.WithCategory("net8").ResolvedId); + } + + [Fact] + public void ExplicitIdsArePreserved() + { + Assert.Equal("net8", Job.Dry.WithId("net8").WithCategory("runtimes").ResolvedId); + } + + // the categories are not needed by the child process, they are used by the host to select the jobs + [Fact] + public void CategoriesAreNotExportedToTheGeneratedSourceCode() + { + var job = Job.Dry.WithLaunchCount(3); + + Assert.Equal( + CharacteristicSetPresenter.SourceCode.ToPresentation(job), + CharacteristicSetPresenter.SourceCode.ToPresentation(job.WithCategories("net8"))); + } + + [Fact] + public void JobsWhichDifferOnlyByCategoriesAreDeduplicatedIntoOne() + { + // Categories are not a part of a job's identity: keeping both would run the same job twice and produce + // two summary rows with the same name. + var mutable = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithCategory("net8")) + .AddJob(Job.Dry.WithCategory("net9")); + + var job = Assert.Single(ImmutableConfigBuilder.Create(mutable).GetJobs()); + + // The survivor carries the categories of all of them, so selecting by either one still matches. + Assert.True(job.Meta.HasCategory("net8")); + Assert.True(job.Meta.HasCategory("net9")); + } + + // Apply overwrites the characteristics, but the categories are additive, so ImmutableConfigBuilder merges + // them explicitly: a mutator that carries categories must not wipe the categories of the job it mutates + [Fact] + public void AMutatorAddsItsCategoriesToTheJobsItMutates() + { + const int warmupCount = 2; + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithCategory("keep")) + .AddJob(Job.Default.WithWarmupCount(warmupCount).WithCategory("mutator").AsMutator()); + + var job = Assert.Single(ImmutableConfigBuilder.Create(config).GetJobs()); + + Assert.Equal(warmupCount, job.Run.WarmupCount); // the mutator was applied + Assert.True(job.Meta.HasCategory("keep")); // ...without dropping what the job already had + Assert.True(job.Meta.HasCategory("mutator")); + } + + [Fact] + public void AMutatorWithoutCategoriesLeavesTheCategoriesOfTheJobsItMutates() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithCategory("keep")) + .AddJob(Job.Default.WithWarmupCount(2).AsMutator()); + + var job = Assert.Single(ImmutableConfigBuilder.Create(config).GetJobs()); + + Assert.Equal(["keep"], job.Meta.Categories); + } + + // UnfreezeCopy is built on Apply, so marking the characteristic as ignored on apply (which would be the + // obvious way to make the categories additive) would silently drop them on every WithXxx call + [Fact] + public void CategoriesSurviveCopyingTheJob() + { + Assert.Equal(["net8"], Job.Dry.WithCategory("net8").UnfreezeCopy().Meta.Categories); + Assert.Equal(["net8", "slow"], Job.Dry.WithCategory("net8").WithLaunchCount(2).WithCategory("slow").Meta.Categories); + } + + [Fact] + public void CategoriesAreNotComparedWhenJobsAreOrdered() + { + Assert.Equal(0, JobComparer.Default.Compare(Job.Dry.WithCategory("a"), Job.Dry.WithCategory("b"))); + Assert.Equal(0, JobComparer.Default.Compare(Job.Dry, Job.Dry.WithCategory("a"))); + } + + [Fact] + public void AnEmptySetOfCategoriesLeavesTheJobUnchanged() + { + var job = Job.Default.WithCategories(); + + // Setting the characteristic to an empty list would mark the job as changed, so a `WithCategories(list)` + // where the list happens to be empty would run the benchmark a second time. + Assert.Empty(job.Meta.Categories); + Assert.False(job.HasValue(MetaMode.CategoriesCharacteristic)); + + var config = ManualConfig.CreateEmpty().AddJob(Job.Default).AddJob(Job.Default.WithCategories()); + Assert.Single(ImmutableConfigBuilder.Create(config).GetJobs()); + } + + [Fact] + public void AnEmptySetOfCategoriesClearsTheCategoriesTheJobAlreadyHad() + { + Assert.Empty(Job.Dry.WithCategory("a").WithCategories().Meta.Categories); + } + + // the guard lives in MetaMode rather than in the WithCategory/WithCategories extensions, because the + // property and AddCategories are public too and would otherwise report a null `source` out of Distinct + [Fact] + public void ANullSetOfCategoriesIsRejectedWhicheverWayItIsPassed() + { + var job = new Job(); + + Assert.Equal("categories", Assert.Throws(() => Job.Default.WithCategories(null!)).ParamName); + Assert.Equal("categories", Assert.Throws(() => job.Meta.Categories = null!).ParamName); + Assert.Equal("categories", Assert.Throws(() => job.Meta.AddCategories(null!)).ParamName); + } + + // a null category matches nothing, survives the merging done when jobs are deduplicated, and only fails + // once something formats it, so it is rejected where it is introduced + [Fact] + public void ANullCategoryIsRejected() + { + Assert.Equal("categories", Assert.Throws(() => Job.Default.WithCategory(null!)).ParamName); + Assert.Equal("categories", Assert.Throws(() => Job.Default.WithCategories("first", null!)).ParamName); + Assert.Equal("categories", Assert.Throws(() => new Job().Meta.AddCategories([null!])).ParamName); + } + + [Fact] + public void EachJobAttributeCarriesItsOwnCategories() + { + var jobs = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoCategorizedJobs)) + .BenchmarksCases + .Select(benchmarkCase => benchmarkCase.Job) + .ToDictionary(job => job.ResolvedId); + + Assert.Equal(2, jobs.Count); + + Assert.True(jobs["first"].Meta.HasCategory("runtimes")); + Assert.True(jobs["first"].Meta.HasCategory("net8")); + Assert.False(jobs["first"].Meta.HasCategory("net9")); + + Assert.True(jobs["second"].Meta.HasCategory("runtimes")); + Assert.True(jobs["second"].Meta.HasCategory("net9")); + Assert.False(jobs["second"].Meta.HasCategory("net8")); + } + + [Fact] + public void CategoriesAreAvailableOnEveryJobAttribute() + { + var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithACategorizedDryJob)).BenchmarksCases.Single().Job; + + Assert.True(job.Meta.HasCategory("debug")); + } + + [Fact] + public void AJobAttributeWithoutCategoriesProducesAJobWithoutCategories() + { + var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithAnUncategorizedJob)).BenchmarksCases.Single().Job; + + Assert.Empty(job.Meta.Categories); + } + + [Fact] + public void ANullCategoriesArgumentIsTreatedAsNoCategories() + { + // `Categories = null` is legal C# for an attribute argument of an array type, so discovery must not + // throw a NullReferenceException on it. + var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithNullCategories)).BenchmarksCases.Single().Job; + + Assert.Empty(job.Meta.Categories); + } + + [Fact] + public void TheCategoriesOfAJobAttributeAreNotPartOfItsId() + { + var categorized = BenchmarkConverter.TypeToBenchmarks(typeof(WithACategorizedDryJob)).BenchmarksCases.Single().Job; + var uncategorized = BenchmarkConverter.TypeToBenchmarks(typeof(WithAnUncategorizedJob)).BenchmarksCases.Single().Job; + + Assert.Equal(uncategorized.ResolvedId, categorized.ResolvedId); + } + + [Fact] + public void TheFilterSelectsBenchmarksByTheCategoriesOfAJobAttribute() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoCategorizedJobs)).BenchmarksCases; + + var matched = benchmarkCases + .Where(benchmarkCase => new JobCategoryFilter(["net9"]).Predicate(benchmarkCase)) + .ToArray(); + + Assert.Equal("second", Assert.Single(matched).Job.ResolvedId); + } + + [Theory] + [InlineData("net8", 1)] + [InlineData("NET8", 1)] // case insensitive + [InlineData("runtimes", 2)] + [InlineData("net", 0)] // it's an exact match, not a substring one + [InlineData("typo", 0)] + public void TheFilterSelectsBenchmarksByJobCategory(string category, int expectedBenchmarks) + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8").WithCategories("net8", "runtimes")) + .AddJob(Job.Dry.WithId("net9").WithCategories("net9", "runtimes")) + .AddJob(Job.Dry.WithId("debug")); + + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithSingleBenchmark), config).BenchmarksCases; + Assert.Equal(3, benchmarkCases.Length); // one per job + + var filter = new JobCategoryFilter([category]); + + Assert.Equal(expectedBenchmarks, benchmarkCases.Count(benchmarkCase => filter.Predicate(benchmarkCase))); + } + + [Fact] + public void MultipleFilterCategoriesAreCombinedWithOr() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8").WithCategory("net8")) + .AddJob(Job.Dry.WithId("net9").WithCategory("net9")) + .AddJob(Job.Dry.WithId("debug").WithCategory("debug")); + + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithSingleBenchmark), config).BenchmarksCases; + + var filter = new JobCategoryFilter(["net8", "net9"]); + var matched = benchmarkCases.Where(benchmarkCase => filter.Predicate(benchmarkCase)).ToArray(); + + Assert.Equal(["net8", "net9"], matched.Select(benchmarkCase => benchmarkCase.Job.ResolvedId).OrderBy(id => id)); + } + + // every category filter is inclusion only, so a job without categories belongs to none of the requested + // ones. This is the documented behaviour: a config mixing categorized and uncategorized jobs loses the + // uncategorized ones as soon as a category is selected. + [Fact] + public void TheFilterExcludesJobsWithoutCategories() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8").WithCategory("net8")) + .AddJob(Job.Dry.WithId("baseline")); + + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithSingleBenchmark), config).BenchmarksCases; + var filter = new JobCategoryFilter(["net8"]); + + Assert.Equal("net8", Assert.Single(benchmarkCases, filter.Predicate).Job.ResolvedId); + } + + [Fact] + public void TheFilterIsAvailableAsAnAttribute() + { + var benchmarkCase = Assert.Single(BenchmarkConverter.TypeToBenchmarks(typeof(WithAFilteredJobCategory)).BenchmarksCases); + + Assert.Equal("second", benchmarkCase.Job.ResolvedId); + } + + [Fact] + public void TheFilterIsAvailableAsAConsoleArgument() + { + var (isSuccess, config, options) = ConfigParser.Parse(["--jobCategories", "net8", "runtimes"], NullLogger.Instance); + + Assert.True(isSuccess); + Assert.True(options!.UserProvidedFilters); + + var filter = Assert.Single(config!.GetFilters().OfType()); + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoCategorizedJobs)).BenchmarksCases; + + Assert.Equal(2, benchmarkCases.Count(filter.Predicate)); // both jobs are in "runtimes" + } + + public class WithSingleBenchmark + { + [Benchmark] public void TheBenchmark() { } + } + + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, id: "first", Categories = ["net8"])] + [SimpleJob(launchCount: 2, warmupCount: 1, iterationCount: 1, id: "second", Categories = ["net9"])] + [JobCategoryFilter("net9")] + public class WithAFilteredJobCategory + { + [Benchmark] public void TheBenchmark() { } + } + + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, id: "first", Categories = ["runtimes", "net8"])] + [SimpleJob(launchCount: 2, warmupCount: 1, iterationCount: 1, id: "second", Categories = ["runtimes", "net9"])] + public class WithTwoCategorizedJobs + { + [Benchmark] public void TheBenchmark() { } + } + + [DryJob(Categories = ["debug"])] + public class WithACategorizedDryJob + { + [Benchmark] public void TheBenchmark() { } + } + + [DryJob] + public class WithAnUncategorizedJob + { + [Benchmark] public void TheBenchmark() { } + } + + [DryJob(Categories = null!)] + public class WithNullCategories + { + [Benchmark] public void TheBenchmark() { } + } + } +}