From 0b1282ecaf74503c0915f65a1c7ad169bb8b0cf4 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sat, 22 Aug 2026 15:03:11 +0100 Subject: [PATCH 01/22] Add job categories Job categories are the job equivalent of [BenchmarkCategory]: they allow grouping jobs so that a subset of them can be selected, without relying on the job Id. - MetaMode.Categories, a hidden characteristic so that categories don't affect the generated job Id, the folder names, the summary nor the code generated for the child process - Job.WithCategory (adds) and Job.WithCategories (overrides), mirroring WithEnvironmentVariable/WithEnvironmentVariables - [JobCategory] attribute, which adds its categories to every job defined for the given class or assembly. It's implemented as a mutator job, so ImmutableConfigBuilder now merges categories instead of overriding them - JobCategoryFilter, which selects the benchmarks of the jobs that belong to any of the given categories --- docs/articles/configs/jobs.md | 30 +++ .../Attributes/Jobs/JobCategoryAttribute.cs | 23 ++ .../Configs/ImmutableConfigBuilder.cs | 7 + .../Filters/JobCategoryFilter.cs | 16 ++ src/BenchmarkDotNet/Jobs/JobExtensions.cs | 18 ++ src/BenchmarkDotNet/Jobs/MetaMode.cs | 36 ++++ .../Jobs/JobCategoryTests.cs | 199 ++++++++++++++++++ 7 files changed, 329 insertions(+) create mode 100644 src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs create mode 100644 src/BenchmarkDotNet/Filters/JobCategoryFilter.cs create mode 100644 tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index 892d9446b4..140472765b 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -15,6 +15,36 @@ There are several categories of characteristics which you can specify. Let's con It's a single string characteristic. It allows to name your job. This name will be used in logs and a part of a folder name with generated files for this job. `Id` doesn't affect benchmark results, but it can be useful for diagnostics. If you don't specify `Id`, random value will be chosen based on other characteristics +### Categories + +Job categories are the job equivalent of `[BenchmarkCategory]`: they allow you to group jobs so that you can select which ones to run. A job can belong to any number of categories, and the comparison is case insensitive. + +```cs +var config = DefaultConfig.Instance + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithCategories("runtimes", "net8")) + .AddJob(Job.Default.WithRuntime(CoreRuntime.Core90).WithCategories("runtimes", "net9")) + .AddJob(Job.Dry.WithCategory("debug")); +``` + +`WithCategories` overrides the categories of the job, `WithCategory` adds to them. + +You can also use the `[JobCategory]` attribute, which adds its categories to every job defined for the given class or assembly: + +```cs +[JobCategory("runtimes")] +[SimpleJob(RuntimeMoniker.Net80)] +[SimpleJob(RuntimeMoniker.Net90)] +public class Benchmarks { /* ... */ } +``` + +Categories don't affect how a job is executed: they are not a part of the job `Id`, the folder names, nor the summary. + +To run only the jobs which belong to given categories, use `JobCategoryFilter`: + +```cs +var config = DefaultConfig.Instance.AddFilter(new JobCategoryFilter(["net8"])); +``` + ### Environment `Environment` specifies an environment of the job. You can specify the following characteristics: diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs new file mode 100644 index 0000000000..a14a0ffd0a --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs @@ -0,0 +1,23 @@ +using BenchmarkDotNet.Jobs; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// Adds the given categories to every job defined for the given class or assembly. + /// It's the job equivalent of . + /// the categories of the jobs which are defined in code are preserved, the categories are added to them + /// + [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] + public class JobCategoryAttribute : JobConfigBaseAttribute + { + // CLS-Compliant Code requires a constructor without an array in the argument list + [PublicAPI] protected JobCategoryAttribute() { } + + public JobCategoryAttribute(params string[] categories) : base(CreateMutatorJob(categories)) { } + + // it's a mutator job so that the categories are applied to all the jobs of the config, + // no matter whether they were defined in code or via other attributes + private static Job CreateMutatorJob(string[] categories) => new Job().WithCategories(categories).AsMutator().Freeze(); + } +} diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 18c1b65fd6..ba42ea083c 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -241,8 +241,15 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) { var copy = result[i].UnfreezeCopy(); + // Apply overrides the characteristics, but the job categories are additive: + // a mutator (eg [JobCategory]) must not drop the categories that the job already has + var ownCategories = copy.Meta.Categories; + copy.Apply(mutatorJob); + if (ownCategories.Count > 0) + copy.Meta.AddCategories(ownCategories); + result[i] = copy.Freeze(); } } diff --git a/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs b/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs new file mode 100644 index 0000000000..33094e6cc4 --- /dev/null +++ b/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs @@ -0,0 +1,16 @@ +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Filters +{ + /// + /// Filter benchmarks which belong to a job that has any of the target job categories + /// + public class JobCategoryFilter : IFilter + { + private readonly string[] targetCategories; + + public JobCategoryFilter(string[] targetCategories) => this.targetCategories = targetCategories; + + public bool Predicate(BenchmarkCase benchmarkCase) => targetCategories.Any(category => benchmarkCase.Job.Meta.HasCategory(category)); + } +} diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 2b5e3ff116..8a83aee1c1 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -324,6 +324,24 @@ public static Job WithMsBuildArguments(this Job job, params string[] msBuildArgu /// public static Job AsDefault(this Job job, bool value = true) => job.WithCore(j => j.Meta.IsDefault = value); + /// + /// Creates a new job based on the given job with the given category added to . + /// categories are metadata used to select which jobs are executed, they don't affect how the job is executed + /// + /// The original job + /// The category which should be added to the new job + /// The new job with the additional category + public static Job WithCategory(this Job job, string category) => job.WithCore(j => j.Meta.AddCategories([category])); + + /// + /// Creates a new job based on the given job with the given categories. + /// the categories of the original job are not preserved, use if you want to add to them + /// + /// The original job + /// The categories of the new job + /// The new job with overriden categories + public static Job WithCategories(this Job job, params string[] categories) => job.WithCore(j => j.Meta.Categories = categories); + internal static Job MakeSettingsUserFriendly(this Job job, Descriptor descriptor) { // users expect that if IterationSetup is configured, it should be run before every benchmark invocation https://github.com/dotnet/BenchmarkDotNet/issues/730 diff --git a/src/BenchmarkDotNet/Jobs/MetaMode.cs b/src/BenchmarkDotNet/Jobs/MetaMode.cs index d57ebe0298..5764defe63 100644 --- a/src/BenchmarkDotNet/Jobs/MetaMode.cs +++ b/src/BenchmarkDotNet/Jobs/MetaMode.cs @@ -9,6 +9,16 @@ public class MetaMode : JobMode [PublicAPI] public static readonly Characteristic IsMutatorCharacteristic = CreateIgnoreOnApplyCharacteristic(nameof(IsMutator)); [PublicAPI] public static readonly Characteristic IsDefaultCharacteristic = CreateHiddenCharacteristic(nameof(IsDefault)); + /// + /// the categories of the job, the job equivalent of + /// + /// the characteristic is hidden on purpose: categories are metadata used to select jobs, they must not affect + /// the generated job id (), the folder names, the summary + /// nor the code generated for the child process + /// + /// + [PublicAPI] public static readonly Characteristic> CategoriesCharacteristic = CreateHiddenCharacteristic>(nameof(Categories)); + public bool Baseline { get => BaselineCharacteristic[this]; @@ -32,5 +42,31 @@ public bool IsDefault get => IsDefaultCharacteristic[this]; set => IsDefaultCharacteristic[this] = value; } + + /// + /// the categories of the job. Setting it overrides the categories that the job already has, + /// use if you want to add to them. + /// + public IReadOnlyList Categories + { + get => CategoriesCharacteristic[this] ?? []; + set => CategoriesCharacteristic[this] = Unique(value); + } + + /// + /// Adds the specified to . + /// The categories that are already present are not duplicated (the comparison is case insensitive). + /// + public void AddCategories(IEnumerable categories) => Categories = [.. Categories, .. categories]; + + /// + /// checks whether the job belongs to given category (the comparison is case insensitive) + /// + public bool HasCategory(string category) => Categories.Contains(category, StringComparer.OrdinalIgnoreCase); + + // the categories are used to select jobs, so we don't want the users to end up with the same category twice + // just because they have used a different casing + private static IReadOnlyList Unique(IEnumerable categories) + => [.. categories.Distinct(StringComparer.OrdinalIgnoreCase)]; } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs new file mode 100644 index 0000000000..10a92359ab --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs @@ -0,0 +1,199 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Filters; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Tests.Jobs +{ + public class JobCategoryTests + { + [Fact] + public void JobsHaveNoCategoriesByDefault() + { + Assert.Empty(Job.Default.Meta.Categories); + Assert.False(Job.Default.Meta.HasCategory("anything")); + } + + [Fact] + public void WithCategoriesOverridesTheExistingCategories() + { + var job = Job.Dry.WithCategories("first").WithCategories("second", "third"); + + Assert.Equal(["second", "third"], job.Meta.Categories); + } + + [Fact] + public void WithCategoryAddsToTheExistingCategories() + { + var job = Job.Dry.WithCategories("first").WithCategory("second"); + + Assert.Equal(["first", "second"], job.Meta.Categories); + } + + [Fact] + public void TheSameCategoryIsNotAddedTwice() + { + var job = Job.Dry.WithCategories("dupe", "DUPE").WithCategory("Dupe"); + + Assert.Equal(["dupe"], job.Meta.Categories); + } + + [Theory] + [InlineData("net8")] + [InlineData("NET8")] + [InlineData("Net8")] + public void HasCategoryIsCaseInsensitive(string category) + { + Assert.True(Job.Dry.WithCategory("net8").Meta.HasCategory(category)); + } + + [Fact] + public void HasCategoryIsAnExactMatch() + { + var job = Job.Dry.WithCategory("net8"); + + Assert.False(job.Meta.HasCategory("net")); + Assert.False(job.Meta.HasCategory("net80")); + } + + [Fact] + public void TheOriginalJobIsNotModified() + { + var original = Job.Dry.WithCategory("first"); + + original.WithCategory("second"); + + Assert.Equal(["first"], original.Meta.Categories); + } + + // categories are metadata, they must not change the identity of the job: + // the id is used for the folder names and the summary, and users don't expect it to change + // just because they have categorized their jobs. See MetaMode.CategoriesCharacteristic + [Fact] + public void CategoriesDoNotAffectTheJobId() + { + var job = Job.Default.WithLaunchCount(3); + var categorized = job.WithCategories("net8", "slow"); + + Assert.Equal(job.ResolvedId, categorized.ResolvedId); + Assert.Equal(job.FolderInfo, categorized.FolderInfo); + Assert.Equal(job.DisplayInfo, categorized.DisplayInfo); + } + + [Fact] + public void CategoriesDoNotAffectTheGeneratedJobIdOfTheDefaultJob() + { + Assert.Equal("DefaultJob", Job.Default.WithCategory("net8").ResolvedId); + } + + [Fact] + public void ExplicitIdsArePreserved() + { + Assert.Equal("net8", Job.Dry.WithId("net8").WithCategory("runtimes").ResolvedId); + } + + // the categories are not needed by the child process, they are used by the host to select the jobs + [Fact] + public void CategoriesAreNotExportedToTheGeneratedSourceCode() + { + var job = Job.Dry.WithLaunchCount(3); + + Assert.Equal( + CharacteristicSetPresenter.SourceCode.ToPresentation(job), + CharacteristicSetPresenter.SourceCode.ToPresentation(job.WithCategories("net8"))); + } + + [Fact] + public void JobsWhichDifferOnlyByCategoriesAreNotConsideredDuplicates() + { + var mutable = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithCategory("net8")) + .AddJob(Job.Dry.WithCategory("net9")); + + var final = ImmutableConfigBuilder.Create(mutable); + + Assert.Equal(2, final.GetJobs().Count()); + } + + [Fact] + public void TheAttributeAddsItsCategoriesToEveryJob() + { + var jobs = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoJobsAndACategory)) + .BenchmarksCases + .Select(benchmarkCase => benchmarkCase.Job) + .ToArray(); + + Assert.Equal(2, jobs.Length); + Assert.All(jobs, job => Assert.True(job.Meta.HasCategory("fromAttribute"))); + } + + [Fact] + public void TheAttributeDoesNotDropTheCategoriesDefinedInCode() + { + var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithCategory("fromCode")); + + var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithACategoryAttribute), config).BenchmarksCases.Single().Job; + + Assert.True(job.Meta.HasCategory("fromCode")); + Assert.True(job.Meta.HasCategory("fromAttribute")); + } + + [Theory] + [InlineData("net8", 1)] + [InlineData("NET8", 1)] // case insensitive + [InlineData("runtimes", 2)] + [InlineData("net", 0)] // it's an exact match, not a substring one + [InlineData("typo", 0)] + public void TheFilterSelectsBenchmarksByJobCategory(string category, int expectedBenchmarks) + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8").WithCategories("net8", "runtimes")) + .AddJob(Job.Dry.WithId("net9").WithCategories("net9", "runtimes")) + .AddJob(Job.Dry.WithId("debug")); + + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithSingleBenchmark), config).BenchmarksCases; + Assert.Equal(3, benchmarkCases.Length); // one per job + + var filter = new JobCategoryFilter([category]); + + Assert.Equal(expectedBenchmarks, benchmarkCases.Count(benchmarkCase => filter.Predicate(benchmarkCase))); + } + + [Fact] + public void MultipleFilterCategoriesAreCombinedWithOr() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8").WithCategory("net8")) + .AddJob(Job.Dry.WithId("net9").WithCategory("net9")) + .AddJob(Job.Dry.WithId("debug").WithCategory("debug")); + + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithSingleBenchmark), config).BenchmarksCases; + + var filter = new JobCategoryFilter(["net8", "net9"]); + var matched = benchmarkCases.Where(benchmarkCase => filter.Predicate(benchmarkCase)).ToArray(); + + Assert.Equal(["net8", "net9"], matched.Select(benchmarkCase => benchmarkCase.Job.ResolvedId).OrderBy(id => id)); + } + + public class WithSingleBenchmark + { + [Benchmark] public void TheBenchmark() { } + } + + [JobCategory("fromAttribute")] + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, id: "first")] + [SimpleJob(launchCount: 2, warmupCount: 1, iterationCount: 1, id: "second")] + public class WithTwoJobsAndACategory + { + [Benchmark] public void TheBenchmark() { } + } + + [JobCategory("fromAttribute")] + public class WithACategoryAttribute + { + [Benchmark] public void TheBenchmark() { } + } + } +} From 3124ad942bfd3556e12599ffa128c125133af25f Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Mon, 24 Aug 2026 08:46:45 +0100 Subject: [PATCH 02/22] Update job category assignment in benchmark examples Replaced [JobCategory] with the Categories property in SimpleJob attributes, allowing multiple categories to be set inline. Updated documentation to clarify that categories do not affect job execution details. --- docs/articles/configs/jobs.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index 140472765b..936b733bf8 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -28,12 +28,11 @@ var config = DefaultConfig.Instance `WithCategories` overrides the categories of the job, `WithCategory` adds to them. -You can also use the `[JobCategory]` attribute, which adds its categories to every job defined for the given class or assembly: +Every job attribute has a `Categories` property, so the same thing can be expressed with attributes: ```cs -[JobCategory("runtimes")] -[SimpleJob(RuntimeMoniker.Net80)] -[SimpleJob(RuntimeMoniker.Net90)] +[SimpleJob(RuntimeMoniker.Net80, Categories = ["runtimes", "net8"])] +[SimpleJob(RuntimeMoniker.Net90, Categories = ["runtimes", "net9"])] public class Benchmarks { /* ... */ } ``` From f7f2da6748e75d1b70c8cfe372978f5d90d10d09 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Mon, 24 Aug 2026 08:47:28 +0100 Subject: [PATCH 03/22] Remove JobCategoryAttribute and related logic Deleted the JobCategoryAttribute.cs file, including all using directives, class definition, constructors, and logic for applying job categories to jobs via attributes. This removes support for assigning categories to jobs through this attribute. --- .../Attributes/Jobs/JobCategoryAttribute.cs | 23 ------------------- 1 file changed, 23 deletions(-) delete mode 100644 src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs deleted file mode 100644 index a14a0ffd0a..0000000000 --- a/src/BenchmarkDotNet/Attributes/Jobs/JobCategoryAttribute.cs +++ /dev/null @@ -1,23 +0,0 @@ -using BenchmarkDotNet.Jobs; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Attributes -{ - /// - /// Adds the given categories to every job defined for the given class or assembly. - /// It's the job equivalent of . - /// the categories of the jobs which are defined in code are preserved, the categories are added to them - /// - [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly, AllowMultiple = true)] - public class JobCategoryAttribute : JobConfigBaseAttribute - { - // CLS-Compliant Code requires a constructor without an array in the argument list - [PublicAPI] protected JobCategoryAttribute() { } - - public JobCategoryAttribute(params string[] categories) : base(CreateMutatorJob(categories)) { } - - // it's a mutator job so that the categories are applied to all the jobs of the config, - // no matter whether they were defined in code or via other attributes - private static Job CreateMutatorJob(string[] categories) => new Job().WithCategories(categories).AsMutator().Freeze(); - } -} From 769d17dacc049c66fa5cc2384a03cec59c57e7fd Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Mon, 24 Aug 2026 08:48:00 +0100 Subject: [PATCH 04/22] Refactor JobConfigBaseAttribute for lazy config & categories Refactor JobConfigBaseAttribute to use lazy config initialization and support job categories. Add a Categories property for job filtering and update constructors to accommodate these changes. Improve comments and code clarity. --- .../Attributes/Jobs/JobConfigbaseAttribute.cs | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs index 0ece453a6e..47883ed0fc 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs @@ -9,13 +9,27 @@ namespace BenchmarkDotNet.Attributes [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)] public class JobConfigBaseAttribute : Attribute, IConfigSource { + private readonly Job? job; + private IConfig? config; + // CLS-Compliant Code requires a constructor which use only CLS-compliant types [PublicAPI] - public JobConfigBaseAttribute() => Config = ManualConfig.CreateEmpty(); + public JobConfigBaseAttribute() { } + + protected JobConfigBaseAttribute(Job job) => this.job = job; - protected JobConfigBaseAttribute(Job job) => Config = ManualConfig.CreateEmpty().AddJob(job); + /// + /// The categories of the job defined by this attribute. Categories are metadata used to select which jobs are + /// executed (see ), they don't affect how the job is executed. + /// + [PublicAPI] public string[] Categories { get; set; } = []; - public IConfig Config { get; } + // Named attribute properties are assigned after the constructor has run, so the config cannot be built there: + // Categories would still be empty. It is read long after the attribute is constructed, so building it lazily + // is enough to see the categories the user has set. + public IConfig Config => config ??= job == null + ? ManualConfig.CreateEmpty() + : ManualConfig.CreateEmpty().AddJob(Categories.Length == 0 ? job : job.WithCategories(Categories).Freeze()); protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? jit, Platform? platform) { From 92f6955b0813f14834da2385ff2f9652ca623101 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Mon, 24 Aug 2026 08:48:18 +0100 Subject: [PATCH 05/22] Remove category preservation when applying mutator jobs Previously, job categories were explicitly preserved and re-added after applying mutator jobs to ensure they were not lost. This logic has been removed, and mutators are now applied directly without restoring categories. --- src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index ba42ea083c..282162c9fe 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -240,16 +240,7 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) for (int i = 0; i < result.Count; i++) { var copy = result[i].UnfreezeCopy(); - - // Apply overrides the characteristics, but the job categories are additive: - // a mutator (eg [JobCategory]) must not drop the categories that the job already has - var ownCategories = copy.Meta.Categories; - copy.Apply(mutatorJob); - - if (ownCategories.Count > 0) - copy.Meta.AddCategories(ownCategories); - result[i] = copy.Freeze(); } } From 1fc3f7fabf228ce9b8b69c1afbe4b7e16f0894c2 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Mon, 24 Aug 2026 08:48:33 +0100 Subject: [PATCH 06/22] Refactor and expand job category tests Replaced WithTwoJobsAndACategory with WithTwoCategorizedJobs using explicit Categories in SimpleJob attributes. Added tests for category assignment, jobs without categories, category-independent job IDs, and category-based filtering. Introduced new test classes and removed obsolete tests and attributes. --- .../Jobs/JobCategoryTests.cs | 70 ++++++++++++++----- 1 file changed, 54 insertions(+), 16 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs index 10a92359ab..46c9467d5d 100644 --- a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs @@ -118,26 +118,59 @@ public void JobsWhichDifferOnlyByCategoriesAreNotConsideredDuplicates() } [Fact] - public void TheAttributeAddsItsCategoriesToEveryJob() + public void EachJobAttributeCarriesItsOwnCategories() { - var jobs = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoJobsAndACategory)) + var jobs = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoCategorizedJobs)) .BenchmarksCases .Select(benchmarkCase => benchmarkCase.Job) - .ToArray(); + .ToDictionary(job => job.ResolvedId); + + Assert.Equal(2, jobs.Count); + + Assert.True(jobs["first"].Meta.HasCategory("runtimes")); + Assert.True(jobs["first"].Meta.HasCategory("net8")); + Assert.False(jobs["first"].Meta.HasCategory("net9")); + + Assert.True(jobs["second"].Meta.HasCategory("runtimes")); + Assert.True(jobs["second"].Meta.HasCategory("net9")); + Assert.False(jobs["second"].Meta.HasCategory("net8")); + } + + [Fact] + public void CategoriesAreAvailableOnEveryJobAttribute() + { + var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithACategorizedDryJob)).BenchmarksCases.Single().Job; - Assert.Equal(2, jobs.Length); - Assert.All(jobs, job => Assert.True(job.Meta.HasCategory("fromAttribute"))); + Assert.True(job.Meta.HasCategory("debug")); } [Fact] - public void TheAttributeDoesNotDropTheCategoriesDefinedInCode() + public void AJobAttributeWithoutCategoriesProducesAJobWithoutCategories() { - var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithCategory("fromCode")); + var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithAnUncategorizedJob)).BenchmarksCases.Single().Job; + + Assert.Empty(job.Meta.Categories); + } - var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithACategoryAttribute), config).BenchmarksCases.Single().Job; + [Fact] + public void TheCategoriesOfAJobAttributeAreNotPartOfItsId() + { + var categorized = BenchmarkConverter.TypeToBenchmarks(typeof(WithACategorizedDryJob)).BenchmarksCases.Single().Job; + var uncategorized = BenchmarkConverter.TypeToBenchmarks(typeof(WithAnUncategorizedJob)).BenchmarksCases.Single().Job; - Assert.True(job.Meta.HasCategory("fromCode")); - Assert.True(job.Meta.HasCategory("fromAttribute")); + Assert.Equal(uncategorized.ResolvedId, categorized.ResolvedId); + } + + [Fact] + public void TheFilterSelectsBenchmarksByTheCategoriesOfAJobAttribute() + { + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoCategorizedJobs)).BenchmarksCases; + + var matched = benchmarkCases + .Where(benchmarkCase => new JobCategoryFilter(["net9"]).Predicate(benchmarkCase)) + .ToArray(); + + Assert.Equal("second", Assert.Single(matched).Job.ResolvedId); } [Theory] @@ -182,16 +215,21 @@ public class WithSingleBenchmark [Benchmark] public void TheBenchmark() { } } - [JobCategory("fromAttribute")] - [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, id: "first")] - [SimpleJob(launchCount: 2, warmupCount: 1, iterationCount: 1, id: "second")] - public class WithTwoJobsAndACategory + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, id: "first", Categories = ["runtimes", "net8"])] + [SimpleJob(launchCount: 2, warmupCount: 1, iterationCount: 1, id: "second", Categories = ["runtimes", "net9"])] + public class WithTwoCategorizedJobs + { + [Benchmark] public void TheBenchmark() { } + } + + [DryJob(Categories = ["debug"])] + public class WithACategorizedDryJob { [Benchmark] public void TheBenchmark() { } } - [JobCategory("fromAttribute")] - public class WithACategoryAttribute + [DryJob] + public class WithAnUncategorizedJob { [Benchmark] public void TheBenchmark() { } } From cd48a26ecccf6ce9ae0537173b095867d1afe13b Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Fri, 28 Aug 2026 09:55:07 +0100 Subject: [PATCH 07/22] Make Categories init-only and nullable in JobConfigbaseAttribute Refactored the Categories property to be init-only and nullable (string[]?), enforcing immutability after initialization. Updated Config property logic to handle null or empty Categories safely. Added remarks to clarify attribute usage and rationale for these changes. --- .../Attributes/Jobs/JobConfigbaseAttribute.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs index 47883ed0fc..94e8f78dbf 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs @@ -22,14 +22,21 @@ public JobConfigBaseAttribute() { } /// The categories of the job defined by this attribute. Categories are metadata used to select which jobs are /// executed (see ), they don't affect how the job is executed. /// - [PublicAPI] public string[] Categories { get; set; } = []; + /// + /// It is init-only because is built once and then kept: a later assignment would be + /// silently ignored. Attribute named arguments are allowed to set init-only properties. + /// + [PublicAPI] public string[]? Categories { get; init; } // Named attribute properties are assigned after the constructor has run, so the config cannot be built there: // Categories would still be empty. It is read long after the attribute is constructed, so building it lazily // is enough to see the categories the user has set. public IConfig Config => config ??= job == null ? ManualConfig.CreateEmpty() - : ManualConfig.CreateEmpty().AddJob(Categories.Length == 0 ? job : job.WithCategories(Categories).Freeze()); + : ManualConfig.CreateEmpty().AddJob( + // `Categories = null` is what an attribute argument of an array type is allowed to be, so it must not + // be dereferenced without a check. + Categories is not { Length: > 0 } categories ? job : job.WithCategories(categories).Freeze()); protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? jit, Platform? platform) { From 335c039437ecf5fb23638ddd24044f248ef243b5 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Fri, 28 Aug 2026 09:55:59 +0100 Subject: [PATCH 08/22] Improve job selection to merge categories correctly Updated job selection logic to group jobs by equality (excluding categories) and merge their categories, replacing Distinct with JobComparer. Added MergeCategories method to ensure all categories are preserved, preventing mismatches when filtering jobs by category. --- .../Configs/ImmutableConfigBuilder.cs | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 282162c9fe..05582324d6 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -223,7 +223,10 @@ private static ImmutableHashSet GetValidators(IEnumerable private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) { - var unique = jobs.Distinct(JobComparer.Default).ToArray(); + // JobComparer ignores the categories, so jobs that differ only by category collapse into one here. Their + // categories are merged into the survivor, otherwise selecting by the categories of the collapsed jobs + // would silently match nothing. + var unique = jobs.GroupBy(job => job, JobComparer.Default).Select(MergeCategories).ToArray(); var result = new List(); foreach (var standardJob in unique.Where(job => !job.Meta.IsMutator && !job.Meta.IsDefault)) @@ -240,7 +243,9 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) for (int i = 0; i < result.Count; i++) { var copy = result[i].UnfreezeCopy(); + copy.Apply(mutatorJob); + result[i] = copy.Freeze(); } } @@ -248,6 +253,19 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) return result; } + /// + /// Returns the first job of a group of jobs which are equal except for their categories, + /// carrying the categories of all of them. + /// + private static Job MergeCategories(IGrouping equalJobs) + { + var first = equalJobs.Key; + var merged = equalJobs.SelectMany(job => job.Meta.Categories).ToArray(); + + // The common case is a group of one, where there is nothing to merge and the job can be returned as it is. + return merged.Length == first.Meta.Categories.Count ? first : first.WithCategories(merged).Freeze(); + } + private class TypeComparer : IEqualityComparer { // different types can implement the same interface, we want to distinct by type From 5afaa87b08acaf255cd4337ad1bcf7a68e28eddf Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Fri, 28 Aug 2026 09:57:09 +0100 Subject: [PATCH 09/22] Skip category in job deduplication comparison The comparison logic for jobs now ignores the MetaMode.CategoriesCharacteristic field. This change prevents jobs that differ only by category from being treated as distinct, ensuring the same job isn't run multiple times under different names. --- src/BenchmarkDotNet/Jobs/JobComparer.cs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 3a46793219..60bb8a1339 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -48,6 +48,12 @@ public int Compare(Job? x, Job? y) foreach (var characteristic in x.GetAllCharacteristics()) { + // Categories say which jobs the user wants to select, they are not a part of what a job *is*. + // Comparing them would let two jobs that differ only by category survive the deduplication done by + // ImmutableConfigBuilder, which would run the same job twice under two identical names. + if (characteristic == MetaMode.CategoriesCharacteristic) + continue; + if (!x.HasValue(characteristic)) { if (y.HasValue(characteristic)) From 02586c10c8b8cd2434dd0986dc1f3f9a8e700ef4 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Fri, 28 Aug 2026 10:12:27 +0100 Subject: [PATCH 10/22] Add null check for categories in WithCategories method Updated the WithCategories extension for Job to throw ArgumentNullException if categories is null. This improves error handling and prevents potential runtime exceptions. --- src/BenchmarkDotNet/Jobs/JobExtensions.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 8a83aee1c1..4d0e4bed12 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -340,7 +340,14 @@ public static Job WithMsBuildArguments(this Job job, params string[] msBuildArgu /// The original job /// The categories of the new job /// The new job with overriden categories - public static Job WithCategories(this Job job, params string[] categories) => job.WithCore(j => j.Meta.Categories = categories); + public static Job WithCategories(this Job job, params string[] categories) + { + // Without this the failure surfaces as an ArgumentNullException for `source`, thrown out of the Distinct + // call which removes the duplicates. + ArgumentNullException.ThrowIfNull(categories); + + return job.WithCore(j => j.Meta.Categories = categories); + } internal static Job MakeSettingsUserFriendly(this Job job, Descriptor descriptor) { From c57dbfda33038b1dfb44c194093fcdd8b95ac9ec Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Fri, 28 Aug 2026 10:14:10 +0100 Subject: [PATCH 11/22] Improve Categories setter to handle empty/null values Updated the Categories property setter to set the value to null if the input is null or results in an empty set after removing duplicates. This change ensures jobs with no categories are treated the same as those never assigned categories, preventing unnecessary updates during category selection. --- src/BenchmarkDotNet/Jobs/MetaMode.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Jobs/MetaMode.cs b/src/BenchmarkDotNet/Jobs/MetaMode.cs index 5764defe63..4290bb1bda 100644 --- a/src/BenchmarkDotNet/Jobs/MetaMode.cs +++ b/src/BenchmarkDotNet/Jobs/MetaMode.cs @@ -50,7 +50,12 @@ public bool IsDefault public IReadOnlyList Categories { get => CategoriesCharacteristic[this] ?? []; - set => CategoriesCharacteristic[this] = Unique(value); + + // Assigning null removes the characteristic (see CharacteristicObject.SetValueCore), which is what an + // empty set of categories has to do: a job that was given no categories must stay indistinguishable from + // one that was never given any, otherwise `WithCategories(selection)` on an empty selection would mark + // the job as changed. + set => CategoriesCharacteristic[this] = Unique(value) is { Count: > 0 } unique ? unique : null!; } /// From c5478c885fcf9c0ce39366ec956cbd88c4910e72 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Fri, 28 Aug 2026 10:14:24 +0100 Subject: [PATCH 12/22] Improve job category tests: deduplication, null, ordering Expanded and clarified the BenchmarkDotNet job category test suite. Renamed and rewrote the deduplication test to verify merging of categories. Added tests for ordering (ignoring categories), handling empty and null categories, and correct exception parameter naming. Introduced a new test class for null category scenarios to ensure robust and correct category handling. --- .../Jobs/JobCategoryTests.cs | 61 ++++++++++++++++++- 1 file changed, 58 insertions(+), 3 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs index 46c9467d5d..c119a99f9c 100644 --- a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs @@ -106,15 +106,54 @@ public void CategoriesAreNotExportedToTheGeneratedSourceCode() } [Fact] - public void JobsWhichDifferOnlyByCategoriesAreNotConsideredDuplicates() + public void JobsWhichDifferOnlyByCategoriesAreDeduplicatedIntoOne() { + // Categories are not a part of a job's identity: keeping both would run the same job twice and produce + // two summary rows with the same name. var mutable = ManualConfig.CreateEmpty() .AddJob(Job.Dry.WithCategory("net8")) .AddJob(Job.Dry.WithCategory("net9")); - var final = ImmutableConfigBuilder.Create(mutable); + var job = Assert.Single(ImmutableConfigBuilder.Create(mutable).GetJobs()); - Assert.Equal(2, final.GetJobs().Count()); + // The survivor carries the categories of all of them, so selecting by either one still matches. + Assert.True(job.Meta.HasCategory("net8")); + Assert.True(job.Meta.HasCategory("net9")); + } + + [Fact] + public void CategoriesAreNotComparedWhenJobsAreOrdered() + { + Assert.Equal(0, JobComparer.Default.Compare(Job.Dry.WithCategory("a"), Job.Dry.WithCategory("b"))); + Assert.Equal(0, JobComparer.Default.Compare(Job.Dry, Job.Dry.WithCategory("a"))); + } + + [Fact] + public void AnEmptySetOfCategoriesLeavesTheJobUnchanged() + { + var job = Job.Default.WithCategories(); + + // Setting the characteristic to an empty list would mark the job as changed, so a `WithCategories(list)` + // where the list happens to be empty would run the benchmark a second time. + Assert.Empty(job.Meta.Categories); + Assert.False(job.HasValue(MetaMode.CategoriesCharacteristic)); + + var config = ManualConfig.CreateEmpty().AddJob(Job.Default).AddJob(Job.Default.WithCategories()); + Assert.Single(ImmutableConfigBuilder.Create(config).GetJobs()); + } + + [Fact] + public void AnEmptySetOfCategoriesClearsTheCategoriesTheJobAlreadyHad() + { + Assert.Empty(Job.Dry.WithCategory("a").WithCategories().Meta.Categories); + } + + [Fact] + public void WithCategoriesNamesTheArgumentItRejects() + { + var exception = Assert.Throws(() => Job.Default.WithCategories(null!)); + + Assert.Equal("categories", exception.ParamName); } [Fact] @@ -152,6 +191,16 @@ public void AJobAttributeWithoutCategoriesProducesAJobWithoutCategories() Assert.Empty(job.Meta.Categories); } + [Fact] + public void ANullCategoriesArgumentIsTreatedAsNoCategories() + { + // `Categories = null` is legal C# for an attribute argument of an array type, so discovery must not + // throw a NullReferenceException on it. + var job = BenchmarkConverter.TypeToBenchmarks(typeof(WithNullCategories)).BenchmarksCases.Single().Job; + + Assert.Empty(job.Meta.Categories); + } + [Fact] public void TheCategoriesOfAJobAttributeAreNotPartOfItsId() { @@ -233,5 +282,11 @@ public class WithAnUncategorizedJob { [Benchmark] public void TheBenchmark() { } } + + [DryJob(Categories = null!)] + public class WithNullCategories + { + [Benchmark] public void TheBenchmark() { } + } } } From ec1681c8b9e99cea5173175aaf974c0b30d1edff Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:17:46 +0100 Subject: [PATCH 13/22] Clarify job category filtering in documentation Updated docs to explain three methods for filtering jobs by category: JobCategoryFilter in code, [JobCategoryFilter] attribute, and --jobCategories argument. Added code and CLI examples. Clarified that filters are inclusion-only and jobs without categories are excluded unless assigned. --- docs/articles/configs/jobs.md | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/docs/articles/configs/jobs.md b/docs/articles/configs/jobs.md index 936b733bf8..2a9abe9fd4 100644 --- a/docs/articles/configs/jobs.md +++ b/docs/articles/configs/jobs.md @@ -38,12 +38,23 @@ public class Benchmarks { /* ... */ } Categories don't affect how a job is executed: they are not a part of the job `Id`, the folder names, nor the summary. -To run only the jobs which belong to given categories, use `JobCategoryFilter`: +To run only the jobs which belong to given categories, use `JobCategoryFilter`, `[JobCategoryFilter]` or the `--jobCategories` console argument: ```cs var config = DefaultConfig.Instance.AddFilter(new JobCategoryFilter(["net8"])); ``` +```cs +[JobCategoryFilter("net8")] +public class Benchmarks { /* ... */ } +``` + +```log +dotnet run -c Release -- --jobCategories net8 +``` + +Like `--anyCategories` and `--allCategories`, the filter is inclusion only: a job with no categories belongs to none of the requested ones, so it is filtered out. If your config mixes categorized and uncategorized jobs, selecting a category also removes the uncategorized jobs, so give a category to every job you may want to keep. + ### Environment `Environment` specifies an environment of the job. You can specify the following characteristics: From 15ae377d04fd33ed676328a37c467359897d1100 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:18:07 +0100 Subject: [PATCH 14/22] Document --jobCategories option in BenchmarkDotNet CLI Updated documentation to describe the new --jobCategories console argument. This option enables running benchmarks for jobs in specified categories, excluding uncategorized jobs. Changes include updates to the usage list and detailed options section. --- docs/articles/guides/console-args.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index eecfb42208..920b0f6c87 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -103,6 +103,7 @@ You can also filter the benchmarks by categories: * `--anyCategories` - runs all benchmarks that belong to **any** of the provided categories * `--allCategories`- runs all benchmarks that belong to **all** provided categories +* `--jobCategories` - runs all benchmarks whose job belongs to **any** of the provided job categories. Jobs without categories are excluded ## Diagnosers @@ -272,6 +273,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `--allStats` (Default: false) Displays all statistics (min, max & more) * `--allCategories` Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed * `--anyCategories` Any Categories to run +* `--jobCategories` Job categories to run. Only the benchmarks which belong to a job that has any of them are going to be executed * `--attribute` Run all methods with given attribute (applied to class or method) * `--join` (Default: false) Prints single table with results for all benchmarks * `--title` Custom title for the produced summaries and the base name of the result files exported for them From cc359221cb40ca13f0204e9037e4c5cbaec99fe2 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:18:31 +0100 Subject: [PATCH 15/22] Add JobCategoryFilterAttribute for benchmark filtering Introduced JobCategoryFilterAttribute in BenchmarkDotNet.Attributes to enable filtering benchmarks by job categories. Includes constructors for CLS compliance and category specification, XML documentation, and PublicAPI annotation. --- .../Filters/JobCategoryFilterAttribute.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs diff --git a/src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs new file mode 100644 index 0000000000..1b55c36151 --- /dev/null +++ b/src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Filters; +using JetBrains.Annotations; + +namespace BenchmarkDotNet.Attributes +{ + /// + /// Runs only the benchmarks whose job belongs to any of the given job categories, + /// see . + /// + public class JobCategoryFilterAttribute : FilterConfigBaseAttribute + { + // CLS-Compliant Code requires a constructor without an array in the argument list + [PublicAPI] + public JobCategoryFilterAttribute() { } + + public JobCategoryFilterAttribute(params string[] targetCategories) : base(new JobCategoryFilter(targetCategories)) { } + } +} From d78e96a1a1fd3310054183a1afdc227497eb3fd4 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:18:46 +0100 Subject: [PATCH 16/22] Preserve job categories when applying mutator jobs Previously, mutating jobs could overwrite existing categories, leading to loss of category information. Now, the original categories are saved, the mutator is applied, and both sets of categories are combined to ensure correct job selection by category. --- src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 05582324d6..7750f77ea4 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -244,8 +244,16 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) { var copy = result[i].UnfreezeCopy(); + // Apply overwrites the characteristics, but the categories are additive: a mutator that carries + // categories (eg [SimpleJob(Categories = ...)] on a method) adds them to every job it mutates + // instead of replacing the ones that job already has. Dropping them would make selecting by the + // categories of the mutated job silently match nothing. + var ownCategories = copy.Meta.Categories; + copy.Apply(mutatorJob); + copy.Meta.Categories = [.. ownCategories, .. copy.Meta.Categories]; + result[i] = copy.Freeze(); } } From edb2b723e5cc26ab1d38a02128c8e101c143a34e Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:19:42 +0100 Subject: [PATCH 17/22] Add JobCategories filter to CommandLineOptions Added JobCategories property to CommandLineOptions for filtering benchmarks by job category. Updated UserProvidedFilters logic to recognize JobCategories as a user-provided filter. --- src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index cc06b11216..1e07cd3685 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -75,6 +75,9 @@ public bool UseDisassemblyDiagnoser [Option("anyCategories", Required = false, HelpText = "Any Categories to run")] public IEnumerable AnyCategories { get; set; } = []; + [Option("jobCategories", Required = false, HelpText = "Job categories to run. Only the benchmarks which belong to a job that has any of them are going to be executed")] + public IEnumerable JobCategories { get; set; } = []; + [Option("attribute", Required = false, HelpText = "Run all methods with given attribute (applied to class or method)")] public IEnumerable AttributeNames { get; set; } = []; @@ -240,7 +243,7 @@ public bool UseDisassemblyDiagnoser [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")] public bool Resume { get; set; } - internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any(); + internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any() || JobCategories.Any(); [Usage(ApplicationAlias = "")] [PublicAPI] From 7694eb9a9cc6987163ce0cab99e410045d76fbb0 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:19:56 +0100 Subject: [PATCH 18/22] Add job category filtering to benchmark config parser Added support for filtering benchmarks by job categories in the configuration parser. If job categories are specified in the options, a JobCategoryFilter is created and applied, enabling selective benchmark execution. --- src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 3c15744439..e787cc6d79 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -797,6 +797,8 @@ private static IEnumerable GetFilters(CommandLineOptions options) yield return new AllCategoriesFilter(options.AllCategories.ToArray()); if (options.AnyCategories.Any()) yield return new AnyCategoriesFilter(options.AnyCategories.ToArray()); + if (options.JobCategories.Any()) + yield return new JobCategoryFilter(options.JobCategories.ToArray()); if (options.AttributeNames.Any()) yield return new AttributesFilter(options.AttributeNames.ToArray()); } From 114b36701b7556c8e02d0f5e6e701cfa448c1358 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:20:09 +0100 Subject: [PATCH 19/22] Clarify JobCategoryFilter remarks in XML docs Added a detailed section to the JobCategoryFilter class summary. The new remarks clarify that the filter is inclusion-only and explain how benchmarks with uncategorized jobs are handled when filtering by category. No code logic was changed. --- src/BenchmarkDotNet/Filters/JobCategoryFilter.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs b/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs index 33094e6cc4..587d2e2026 100644 --- a/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs +++ b/src/BenchmarkDotNet/Filters/JobCategoryFilter.cs @@ -3,7 +3,12 @@ namespace BenchmarkDotNet.Filters { /// - /// Filter benchmarks which belong to a job that has any of the target job categories + /// Filter benchmarks which belong to a job that has any of the target job categories. + /// + /// Like every other category filter, this one is inclusion only: a benchmark whose job has no categories at all + /// belongs to none of the target categories, so it is filtered out. In a config that mixes categorized and + /// uncategorized jobs, selecting a category therefore also removes the uncategorized jobs. + /// /// public class JobCategoryFilter : IFilter { From 65181d9a3c22affab5d1d6e3dd1085064b537afc Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:20:23 +0100 Subject: [PATCH 20/22] Refactor WithCategories to expression-bodied method Refactored the WithCategories method in BenchmarkDotNet.Jobs to a single-line expression-bodied method, removing the explicit ArgumentNullException check for the categories parameter. --- src/BenchmarkDotNet/Jobs/JobExtensions.cs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 4d0e4bed12..8a83aee1c1 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -340,14 +340,7 @@ public static Job WithMsBuildArguments(this Job job, params string[] msBuildArgu /// The original job /// The categories of the new job /// The new job with overriden categories - public static Job WithCategories(this Job job, params string[] categories) - { - // Without this the failure surfaces as an ArgumentNullException for `source`, thrown out of the Distinct - // call which removes the duplicates. - ArgumentNullException.ThrowIfNull(categories); - - return job.WithCore(j => j.Meta.Categories = categories); - } + public static Job WithCategories(this Job job, params string[] categories) => job.WithCore(j => j.Meta.Categories = categories); internal static Job MakeSettingsUserFriendly(this Job job, Descriptor descriptor) { From ffc73de0b3317a041220e41659e1d7c6a17ab49b Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:20:35 +0100 Subject: [PATCH 21/22] Enhance validation for job categories in MetaMode Improved validation and error handling in BenchmarkDotNet.Jobs.MetaMode. AddCategories now throws ArgumentNullException for null input. Unique method checks for null input and null categories, throwing exceptions as needed. Updated comments to clarify validation and merging logic. --- src/BenchmarkDotNet/Jobs/MetaMode.cs | 35 +++++++++++++++++++++++----- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/BenchmarkDotNet/Jobs/MetaMode.cs b/src/BenchmarkDotNet/Jobs/MetaMode.cs index 4290bb1bda..f5e75897ba 100644 --- a/src/BenchmarkDotNet/Jobs/MetaMode.cs +++ b/src/BenchmarkDotNet/Jobs/MetaMode.cs @@ -14,7 +14,10 @@ public class MetaMode : JobMode /// /// the characteristic is hidden on purpose: categories are metadata used to select jobs, they must not affect /// the generated job id (), the folder names, the summary - /// nor the code generated for the child process + /// nor the code generated for the child process. + /// it is not ignored on apply, even though categories are additive rather than overwritten: UnfreezeCopy is + /// built on Apply, so an ignored characteristic would be dropped by every WithXxx call. The places that have + /// to add rather than overwrite merge the categories explicitly, see /// /// [PublicAPI] public static readonly Characteristic> CategoriesCharacteristic = CreateHiddenCharacteristic>(nameof(Categories)); @@ -62,16 +65,36 @@ public IReadOnlyList Categories /// Adds the specified to . /// The categories that are already present are not duplicated (the comparison is case insensitive). /// - public void AddCategories(IEnumerable categories) => Categories = [.. Categories, .. categories]; + public void AddCategories(IEnumerable categories) + { + ArgumentNullException.ThrowIfNull(categories); + + Categories = [.. Categories, .. categories]; + } /// /// checks whether the job belongs to given category (the comparison is case insensitive) /// public bool HasCategory(string category) => Categories.Contains(category, StringComparer.OrdinalIgnoreCase); - // the categories are used to select jobs, so we don't want the users to end up with the same category twice - // just because they have used a different casing + // Every category assigned to a job goes through here, which is why the arguments are validated here rather + // than in the WithCategory/WithCategories extensions: the property and AddCategories are public too, and + // guarding only the extensions would leave `job.Meta.Categories = null` reported as a null `source` thrown + // out of Distinct, and would let a null category be stored and only fail much later. private static IReadOnlyList Unique(IEnumerable categories) - => [.. categories.Distinct(StringComparer.OrdinalIgnoreCase)]; + { + ArgumentNullException.ThrowIfNull(categories); + + var all = categories.ToArray(); + + // A null category can never be selected and it survives the merging done when jobs are deduplicated, + // so it would surface far away from the call that introduced it. + if (all.Any(category => category is null)) + throw new ArgumentException("A job category must not be null.", nameof(categories)); + + // the categories are used to select jobs, so we don't want the users to end up with the same category + // twice just because they have used a different casing + return [.. all.Distinct(StringComparer.OrdinalIgnoreCase)]; + } } -} \ No newline at end of file +} From f86265d39b00d035ff762939f632caa092d9a528 Mon Sep 17 00:00:00 2001 From: Ifeanyi Shadrach Odom Date: Sun, 30 Aug 2026 19:20:49 +0100 Subject: [PATCH 22/22] Enhance JobCategoryTests with comprehensive scenarios Added extensive tests to JobCategoryTests.cs covering category merging, handling of null/empty categories, and category filtering via attributes and console arguments. Ensured categories are additive, invalid inputs are rejected, and introduced supporting test classes and attributes. --- .../Jobs/JobCategoryTests.cs | 106 +++++++++++++++++- 1 file changed, 103 insertions(+), 3 deletions(-) diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs index c119a99f9c..926775aa88 100644 --- a/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobCategoryTests.cs @@ -1,8 +1,10 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Configs; +using BenchmarkDotNet.ConsoleArguments; using BenchmarkDotNet.Filters; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; namespace BenchmarkDotNet.Tests.Jobs @@ -121,6 +123,44 @@ public void JobsWhichDifferOnlyByCategoriesAreDeduplicatedIntoOne() Assert.True(job.Meta.HasCategory("net9")); } + // Apply overwrites the characteristics, but the categories are additive, so ImmutableConfigBuilder merges + // them explicitly: a mutator that carries categories must not wipe the categories of the job it mutates + [Fact] + public void AMutatorAddsItsCategoriesToTheJobsItMutates() + { + const int warmupCount = 2; + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithCategory("keep")) + .AddJob(Job.Default.WithWarmupCount(warmupCount).WithCategory("mutator").AsMutator()); + + var job = Assert.Single(ImmutableConfigBuilder.Create(config).GetJobs()); + + Assert.Equal(warmupCount, job.Run.WarmupCount); // the mutator was applied + Assert.True(job.Meta.HasCategory("keep")); // ...without dropping what the job already had + Assert.True(job.Meta.HasCategory("mutator")); + } + + [Fact] + public void AMutatorWithoutCategoriesLeavesTheCategoriesOfTheJobsItMutates() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithCategory("keep")) + .AddJob(Job.Default.WithWarmupCount(2).AsMutator()); + + var job = Assert.Single(ImmutableConfigBuilder.Create(config).GetJobs()); + + Assert.Equal(["keep"], job.Meta.Categories); + } + + // UnfreezeCopy is built on Apply, so marking the characteristic as ignored on apply (which would be the + // obvious way to make the categories additive) would silently drop them on every WithXxx call + [Fact] + public void CategoriesSurviveCopyingTheJob() + { + Assert.Equal(["net8"], Job.Dry.WithCategory("net8").UnfreezeCopy().Meta.Categories); + Assert.Equal(["net8", "slow"], Job.Dry.WithCategory("net8").WithLaunchCount(2).WithCategory("slow").Meta.Categories); + } + [Fact] public void CategoriesAreNotComparedWhenJobsAreOrdered() { @@ -148,12 +188,26 @@ public void AnEmptySetOfCategoriesClearsTheCategoriesTheJobAlreadyHad() Assert.Empty(Job.Dry.WithCategory("a").WithCategories().Meta.Categories); } + // the guard lives in MetaMode rather than in the WithCategory/WithCategories extensions, because the + // property and AddCategories are public too and would otherwise report a null `source` out of Distinct [Fact] - public void WithCategoriesNamesTheArgumentItRejects() + public void ANullSetOfCategoriesIsRejectedWhicheverWayItIsPassed() { - var exception = Assert.Throws(() => Job.Default.WithCategories(null!)); + var job = new Job(); - Assert.Equal("categories", exception.ParamName); + Assert.Equal("categories", Assert.Throws(() => Job.Default.WithCategories(null!)).ParamName); + Assert.Equal("categories", Assert.Throws(() => job.Meta.Categories = null!).ParamName); + Assert.Equal("categories", Assert.Throws(() => job.Meta.AddCategories(null!)).ParamName); + } + + // a null category matches nothing, survives the merging done when jobs are deduplicated, and only fails + // once something formats it, so it is rejected where it is introduced + [Fact] + public void ANullCategoryIsRejected() + { + Assert.Equal("categories", Assert.Throws(() => Job.Default.WithCategory(null!)).ParamName); + Assert.Equal("categories", Assert.Throws(() => Job.Default.WithCategories("first", null!)).ParamName); + Assert.Equal("categories", Assert.Throws(() => new Job().Meta.AddCategories([null!])).ParamName); } [Fact] @@ -259,11 +313,57 @@ public void MultipleFilterCategoriesAreCombinedWithOr() Assert.Equal(["net8", "net9"], matched.Select(benchmarkCase => benchmarkCase.Job.ResolvedId).OrderBy(id => id)); } + // every category filter is inclusion only, so a job without categories belongs to none of the requested + // ones. This is the documented behaviour: a config mixing categorized and uncategorized jobs loses the + // uncategorized ones as soon as a category is selected. + [Fact] + public void TheFilterExcludesJobsWithoutCategories() + { + var config = ManualConfig.CreateEmpty() + .AddJob(Job.Dry.WithId("net8").WithCategory("net8")) + .AddJob(Job.Dry.WithId("baseline")); + + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithSingleBenchmark), config).BenchmarksCases; + var filter = new JobCategoryFilter(["net8"]); + + Assert.Equal("net8", Assert.Single(benchmarkCases, filter.Predicate).Job.ResolvedId); + } + + [Fact] + public void TheFilterIsAvailableAsAnAttribute() + { + var benchmarkCase = Assert.Single(BenchmarkConverter.TypeToBenchmarks(typeof(WithAFilteredJobCategory)).BenchmarksCases); + + Assert.Equal("second", benchmarkCase.Job.ResolvedId); + } + + [Fact] + public void TheFilterIsAvailableAsAConsoleArgument() + { + var (isSuccess, config, options) = ConfigParser.Parse(["--jobCategories", "net8", "runtimes"], NullLogger.Instance); + + Assert.True(isSuccess); + Assert.True(options!.UserProvidedFilters); + + var filter = Assert.Single(config!.GetFilters().OfType()); + var benchmarkCases = BenchmarkConverter.TypeToBenchmarks(typeof(WithTwoCategorizedJobs)).BenchmarksCases; + + Assert.Equal(2, benchmarkCases.Count(filter.Predicate)); // both jobs are in "runtimes" + } + public class WithSingleBenchmark { [Benchmark] public void TheBenchmark() { } } + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, id: "first", Categories = ["net8"])] + [SimpleJob(launchCount: 2, warmupCount: 1, iterationCount: 1, id: "second", Categories = ["net9"])] + [JobCategoryFilter("net9")] + public class WithAFilteredJobCategory + { + [Benchmark] public void TheBenchmark() { } + } + [SimpleJob(launchCount: 1, warmupCount: 1, iterationCount: 1, id: "first", Categories = ["runtimes", "net8"])] [SimpleJob(launchCount: 2, warmupCount: 1, iterationCount: 1, id: "second", Categories = ["runtimes", "net9"])] public class WithTwoCategorizedJobs