Add job categories

docs/articles/configs/jobs.md

@@ -15,6 +15,46 @@ There are several categories of characteristics which you can specify. Let's con
 
 It's a single string characteristic. It allows to name your job. This name will be used in logs and a part of a folder name with generated files for this job. `Id` doesn't affect benchmark results, but it can be useful for diagnostics. If you don't specify `Id`, random value will be chosen based on other characteristics
 
+### Categories
+
+Job categories are the job equivalent of `[BenchmarkCategory]`: they allow you to group jobs so that you can select which ones to run. A job can belong to any number of categories, and the comparison is case insensitive.
+
+```cs
+var config = DefaultConfig.Instance
+    .AddJob(Job.Default.WithRuntime(CoreRuntime.Core80).WithCategories("runtimes", "net8"))
+    .AddJob(Job.Default.WithRuntime(CoreRuntime.Core90).WithCategories("runtimes", "net9"))
+    .AddJob(Job.Dry.WithCategory("debug"));
+```
+
+`WithCategories` overrides the categories of the job, `WithCategory` adds to them.
+
+Every job attribute has a `Categories` property, so the same thing can be expressed with attributes:
+
+```cs
+[SimpleJob(RuntimeMoniker.Net80, Categories = ["runtimes", "net8"])]
+[SimpleJob(RuntimeMoniker.Net90, Categories = ["runtimes", "net9"])]
+public class Benchmarks { /* ... */ }
+```
+
+Categories don't affect how a job is executed: they are not a part of the job `Id`, the folder names, nor the summary.
+
+To run only the jobs which belong to given categories, use `JobCategoryFilter`, `[JobCategoryFilter]` or the `--jobCategories` console argument:
+
+```cs
+var config = DefaultConfig.Instance.AddFilter(new JobCategoryFilter(["net8"]));
+```
+
+```cs
+[JobCategoryFilter("net8")]
+public class Benchmarks { /* ... */ }
+```
+
+```log
+dotnet run -c Release -- --jobCategories net8
+```
+
+Like `--anyCategories` and `--allCategories`, the filter is inclusion only: a job with no categories belongs to none of the requested ones, so it is filtered out. If your config mixes categorized and uncategorized jobs, selecting a category also removes the uncategorized jobs, so give a category to every job you may want to keep.
+
 ### Environment
 
 `Environment` specifies an environment of the job. You can specify the following characteristics:

docs/articles/guides/console-args.md

@@ -103,6 +103,7 @@ You can also filter the benchmarks by categories:
 
 * `--anyCategories` - runs all benchmarks that belong to **any** of the provided categories
 * `--allCategories`- runs all benchmarks that belong to **all** provided categories
+* `--jobCategories` - runs all benchmarks whose job belongs to **any** of the provided job categories. Jobs without categories are excluded
 
 ## Diagnosers
 
@@ -272,6 +273,7 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5
 * `--allStats`                (Default: false) Displays all statistics (min, max & more)
 * `--allCategories`           Categories to run. If few are provided, only the benchmarks which belong to all of them are going to be executed
 * `--anyCategories`           Any Categories to run
+* `--jobCategories`           Job categories to run. Only the benchmarks which belong to a job that has any of them are going to be executed
 * `--attribute`               Run all methods with given attribute (applied to class or method)
 * `--join`                    (Default: false) Prints single table with results for all benchmarks
 * `--title`                   Custom title for the produced summaries and the base name of the result files exported for them

src/BenchmarkDotNet/Attributes/Filters/JobCategoryFilterAttribute.cs

@@ -0,0 +1,18 @@
+using BenchmarkDotNet.Filters;
+using JetBrains.Annotations;
+
+namespace BenchmarkDotNet.Attributes
+{
+    /// <summary>
+    /// Runs only the benchmarks whose job belongs to any of the given job categories,
+    /// see <see cref="JobCategoryFilter"/>.
+    /// </summary>
+    public class JobCategoryFilterAttribute : FilterConfigBaseAttribute
+    {
+        // CLS-Compliant Code requires a constructor without an array in the argument list
+        [PublicAPI]
+        public JobCategoryFilterAttribute() { }
+
+        public JobCategoryFilterAttribute(params string[] targetCategories) : base(new JobCategoryFilter(targetCategories)) { }
+    }
+}

src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs

@@ -9,13 +9,34 @@ namespace BenchmarkDotNet.Attributes
     [AttributeUsage(AttributeTargets.Class | AttributeTargets.Assembly)]
     public class JobConfigBaseAttribute : Attribute, IConfigSource
     {
+        private readonly Job? job;
+        private IConfig? config;
+
         // CLS-Compliant Code requires a constructor which use only CLS-compliant types
         [PublicAPI]
-        public JobConfigBaseAttribute() => Config = ManualConfig.CreateEmpty();
+        public JobConfigBaseAttribute() { }
+
+        protected JobConfigBaseAttribute(Job job) => this.job = job;
 
-        protected JobConfigBaseAttribute(Job job) => Config = ManualConfig.CreateEmpty().AddJob(job);
+        /// <summary>
+        /// The categories of the job defined by this attribute. Categories are metadata used to select which jobs are
+        /// executed (see <see cref="BenchmarkDotNet.Filters.JobCategoryFilter"/>), they don't affect how the job is executed.
+        /// </summary>
+        /// <remarks>
+        /// It is init-only because <see cref="Config"/> is built once and then kept: a later assignment would be
+        /// silently ignored. Attribute named arguments are allowed to set init-only properties.
+        /// </remarks>
+        [PublicAPI] public string[]? Categories { get; init; }
 
-        public IConfig Config { get; }
+        // Named attribute properties are assigned after the constructor has run, so the config cannot be built there:
+        // Categories would still be empty. It is read long after the attribute is constructed, so building it lazily
+        // is enough to see the categories the user has set.
+        public IConfig Config => config ??= job == null
+            ? ManualConfig.CreateEmpty()
+            : ManualConfig.CreateEmpty().AddJob(
+                // `Categories = null` is what an attribute argument of an array type is allowed to be, so it must not
+                // be dereferenced without a check.
+                Categories is not { Length: > 0 } categories ? job : job.WithCategories(categories).Freeze());
 
         protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? jit, Platform? platform)
         {

src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs

@@ -223,7 +223,10 @@ private static ImmutableHashSet<IValidator> GetValidators(IEnumerable<IValidator
         /// </summary>
         private static IReadOnlyList<Job> GetRunnableJobs(IEnumerable<Job> jobs)
         {
-            var unique = jobs.Distinct(JobComparer.Default).ToArray();
+            // JobComparer ignores the categories, so jobs that differ only by category collapse into one here. Their
+            // categories are merged into the survivor, otherwise selecting by the categories of the collapsed jobs
+            // would silently match nothing.
+            var unique = jobs.GroupBy(job => job, JobComparer.Default).Select(MergeCategories).ToArray();
             var result = new List<Job>();
 
             foreach (var standardJob in unique.Where(job => !job.Meta.IsMutator && !job.Meta.IsDefault))
@@ -241,15 +244,36 @@ private static IReadOnlyList<Job> GetRunnableJobs(IEnumerable<Job> jobs)
                 {
                     var copy = result[i].UnfreezeCopy();
 
+                    // Apply overwrites the characteristics, but the categories are additive: a mutator that carries
+                    // categories (eg [SimpleJob(Categories = ...)] on a method) adds them to every job it mutates
+                    // instead of replacing the ones that job already has. Dropping them would make selecting by the
+                    // categories of the mutated job silently match nothing.
+                    var ownCategories = copy.Meta.Categories;
+
                     copy.Apply(mutatorJob);
 
+                    copy.Meta.Categories = [.. ownCategories, .. copy.Meta.Categories];
+
                     result[i] = copy.Freeze();
                 }
             }
 
             return result;
         }
 
+        /// <summary>
+        /// Returns the first job of a group of jobs which are equal except for their categories,
+        /// carrying the categories of all of them.
+        /// </summary>
+        private static Job MergeCategories(IGrouping<Job, Job> equalJobs)
+        {
+            var first = equalJobs.Key;
+            var merged = equalJobs.SelectMany(job => job.Meta.Categories).ToArray();
+
+            // The common case is a group of one, where there is nothing to merge and the job can be returned as it is.
+            return merged.Length == first.Meta.Categories.Count ? first : first.WithCategories(merged).Freeze();
+        }
+
         private class TypeComparer<TInterface> : IEqualityComparer<TInterface>
         {
             // different types can implement the same interface, we want to distinct by type

src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs

@@ -75,6 +75,9 @@ public bool UseDisassemblyDiagnoser
         [Option("anyCategories", Required = false, HelpText = "Any Categories to run")]
         public IEnumerable<string> AnyCategories { get; set; } = [];
 
+        [Option("jobCategories", Required = false, HelpText = "Job categories to run. Only the benchmarks which belong to a job that has any of them are going to be executed")]
+        public IEnumerable<string> JobCategories { get; set; } = [];
+
         [Option("attribute", Required = false, HelpText = "Run all methods with given attribute (applied to class or method)")]
         public IEnumerable<string> AttributeNames { get; set; } = [];
 
@@ -240,7 +243,7 @@ public bool UseDisassemblyDiagnoser
         [Option("resume", Required = false, Default = false, HelpText = "Continue the execution if the last run was stopped.")]
         public bool Resume { get; set; }
 
-        internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any();
+        internal bool UserProvidedFilters => Filters.Any() || AttributeNames.Any() || AllCategories.Any() || AnyCategories.Any() || JobCategories.Any();
 
         [Usage(ApplicationAlias = "")]
         [PublicAPI]

src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs

@@ -797,6 +797,8 @@ private static IEnumerable<IFilter> GetFilters(CommandLineOptions options)
                 yield return new AllCategoriesFilter(options.AllCategories.ToArray());
             if (options.AnyCategories.Any())
                 yield return new AnyCategoriesFilter(options.AnyCategories.ToArray());
+            if (options.JobCategories.Any())
+                yield return new JobCategoryFilter(options.JobCategories.ToArray());
             if (options.AttributeNames.Any())
                 yield return new AttributesFilter(options.AttributeNames.ToArray());
         }

src/BenchmarkDotNet/Filters/JobCategoryFilter.cs

@@ -0,0 +1,21 @@
+using BenchmarkDotNet.Running;
+
+namespace BenchmarkDotNet.Filters
+{
+    /// <summary>
+    /// Filter benchmarks which belong to a job that has any of the target job categories.
+    /// <remarks>
+    /// Like every other category filter, this one is inclusion only: a benchmark whose job has no categories at all
+    /// belongs to none of the target categories, so it is filtered out. In a config that mixes categorized and
+    /// uncategorized jobs, selecting a category therefore also removes the uncategorized jobs.
+    /// </remarks>
+    /// </summary>
+    public class JobCategoryFilter : IFilter
+    {
+        private readonly string[] targetCategories;
+
+        public JobCategoryFilter(string[] targetCategories) => this.targetCategories = targetCategories;
+
+        public bool Predicate(BenchmarkCase benchmarkCase) => targetCategories.Any(category => benchmarkCase.Job.Meta.HasCategory(category));
+    }
+}

src/BenchmarkDotNet/Jobs/JobComparer.cs

@@ -48,6 +48,12 @@ public int Compare(Job? x, Job? y)
 
             foreach (var characteristic in x.GetAllCharacteristics())
             {
+                // Categories say which jobs the user wants to select, they are not a part of what a job *is*.
+                // Comparing them would let two jobs that differ only by category survive the deduplication done by
+                // ImmutableConfigBuilder, which would run the same job twice under two identical names.
+                if (characteristic == MetaMode.CategoriesCharacteristic)
+                    continue;
+
                 if (!x.HasValue(characteristic))
                 {
                     if (y.HasValue(characteristic))

src/BenchmarkDotNet/Jobs/JobExtensions.cs

@@ -324,6 +324,24 @@ public static Job WithMsBuildArguments(this Job job, params string[] msBuildArgu
         /// </summary>
         public static Job AsDefault(this Job job, bool value = true) => job.WithCore(j => j.Meta.IsDefault = value);
 
+        /// <summary>
+        /// Creates a new job based on the given job with the given category added to <see cref="MetaMode.Categories"/>.
+        /// <remarks>categories are metadata used to select which jobs are executed, they don't affect how the job is executed</remarks>
+        /// </summary>
+        /// <param name="job">The original job</param>
+        /// <param name="category">The category which should be added to the new job</param>
+        /// <returns>The new job with the additional category</returns>
+        public static Job WithCategory(this Job job, string category) => job.WithCore(j => j.Meta.AddCategories([category]));
+
+        /// <summary>
+        /// Creates a new job based on the given job with the given categories.
+        /// <remarks>the categories of the original job are not preserved, use <see cref="WithCategory"/> if you want to add to them</remarks>
+        /// </summary>
+        /// <param name="job">The original job</param>
+        /// <param name="categories">The categories of the new job</param>
+        /// <returns>The new job with overriden categories</returns>
+        public static Job WithCategories(this Job job, params string[] categories) => job.WithCore(j => j.Meta.Categories = categories);
+
         internal static Job MakeSettingsUserFriendly(this Job job, Descriptor descriptor)
         {
             // users expect that if IterationSetup is configured, it should be run before every benchmark invocation https://github.com/dotnet/BenchmarkDotNet/issues/730

src/BenchmarkDotNet/Jobs/MetaMode.cs

@@ -9,6 +9,19 @@ public class MetaMode : JobMode<MetaMode>
         [PublicAPI] public static readonly Characteristic<bool> IsMutatorCharacteristic = CreateIgnoreOnApplyCharacteristic<bool>(nameof(IsMutator));
         [PublicAPI] public static readonly Characteristic<bool> IsDefaultCharacteristic = CreateHiddenCharacteristic<bool>(nameof(IsDefault));
 
+        /// <summary>
+        /// the categories of the job, the job equivalent of <see cref="Attributes.BenchmarkCategoryAttribute"/>
+        /// <remarks>
+        /// the characteristic is hidden on purpose: categories are metadata used to select jobs, they must not affect
+        /// the generated job id (<see cref="JobIdGenerator"/>), the folder names, the summary
+        /// nor the code generated for the child process.
+        /// it is not ignored on apply, even though categories are additive rather than overwritten: UnfreezeCopy is
+        /// built on Apply, so an ignored characteristic would be dropped by every WithXxx call. The places that have
+        /// to add rather than overwrite merge the categories explicitly, see <see cref="Configs.ImmutableConfigBuilder"/>
+        /// </remarks>
+        /// </summary>
+        [PublicAPI] public static readonly Characteristic<IReadOnlyList<string>> CategoriesCharacteristic = CreateHiddenCharacteristic<IReadOnlyList<string>>(nameof(Categories));
+
         public bool Baseline
         {
             get => BaselineCharacteristic[this];
@@ -32,5 +45,56 @@ public bool IsDefault
             get => IsDefaultCharacteristic[this];
             set => IsDefaultCharacteristic[this] = value;
         }
+
+        /// <summary>
+        /// the categories of the job. Setting it overrides the categories that the job already has,
+        /// use <see cref="AddCategories"/> if you want to add to them.
+        /// </summary>
+        public IReadOnlyList<string> Categories
+        {
+            get => CategoriesCharacteristic[this] ?? [];
+
+            // Assigning null removes the characteristic (see CharacteristicObject.SetValueCore), which is what an
+            // empty set of categories has to do: a job that was given no categories must stay indistinguishable from
+            // one that was never given any, otherwise `WithCategories(selection)` on an empty selection would mark
+            // the job as changed.
+            set => CategoriesCharacteristic[this] = Unique(value) is { Count: > 0 } unique ? unique : null!;
+        }
+
+        /// <summary>
+        /// Adds the specified <paramref name="categories"/> to <see cref="Categories"/>.
+        /// The categories that are already present are not duplicated (the comparison is case insensitive).
+        /// </summary>
+        public void AddCategories(IEnumerable<string> categories)
+        {
+            ArgumentNullException.ThrowIfNull(categories);
+
+            Categories = [.. Categories, .. categories];
+        }
+
+        /// <summary>
+        /// checks whether the job belongs to given category (the comparison is case insensitive)
+        /// </summary>
+        public bool HasCategory(string category) => Categories.Contains(category, StringComparer.OrdinalIgnoreCase);
+
+        // Every category assigned to a job goes through here, which is why the arguments are validated here rather
+        // than in the WithCategory/WithCategories extensions: the property and AddCategories are public too, and
+        // guarding only the extensions would leave `job.Meta.Categories = null` reported as a null `source` thrown
+        // out of Distinct, and would let a null category be stored and only fail much later.
+        private static IReadOnlyList<string> Unique(IEnumerable<string> categories)
+        {
+            ArgumentNullException.ThrowIfNull(categories);
+
+            var all = categories.ToArray();
+
+            // A null category can never be selected and it survives the merging done when jobs are deduplicated,
+            // so it would surface far away from the call that introduced it.
+            if (all.Any(category => category is null))
+                throw new ArgumentException("A job category must not be null.", nameof(categories));
+
+            // the categories are used to select jobs, so we don't want the users to end up with the same category
+            // twice just because they have used a different casing
+            return [.. all.Distinct(StringComparer.OrdinalIgnoreCase)];
+        }
     }
-}
+}