diff --git a/.github/workflows/run-tests-selected.yaml b/.github/workflows/run-tests-selected.yaml index 0e2f436810..e4e8be56fe 100644 --- a/.github/workflows/run-tests-selected.yaml +++ b/.github/workflows/run-tests-selected.yaml @@ -26,6 +26,10 @@ on: - tests/BenchmarkDotNet.IntegrationTests - tests/BenchmarkDotNet.IntegrationTests.ManualRunning - tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks + # This one is a Microsoft.Testing.Platform application. It carries the `global.json` which routes + # `dotnet test` to the platform, and that file is resolved from the working directory, which this + # workflow sets to the project. + - tests/BenchmarkDotNet.IntegrationTests.TestingPlatform - samples/BenchmarkDotNet.Samples framework: type: choice diff --git a/.github/workflows/run-tests.yaml b/.github/workflows/run-tests.yaml index b3ea72e473..abc15b37dd 100644 --- a/.github/workflows/run-tests.yaml +++ b/.github/workflows/run-tests.yaml @@ -185,6 +185,11 @@ jobs: - uses: actions/checkout@v7 - name: Run task 'pack' run: ./build.cmd pack + # Nothing in the solution consumes BenchmarkDotNet.TestAdapter as a package, so nothing else exercises the + # order in which NuGet imports its build files relative to Microsoft.Testing.Platform.MSBuild's. + - name: Smoke test the packed BenchmarkDotNet.TestAdapter + shell: pwsh + run: ./build/smoke-tests/test-adapter-consumer.ps1 spellcheck-docs: runs-on: ubuntu-latest diff --git a/BenchmarkDotNet.slnx b/BenchmarkDotNet.slnx index 2d5a3cf0dd..f6801feaba 100644 --- a/BenchmarkDotNet.slnx +++ b/BenchmarkDotNet.slnx @@ -42,6 +42,8 @@ + + diff --git a/build/cSpell.json b/build/cSpell.json index e90695fa7f..db68aadb88 100644 --- a/build/cSpell.json +++ b/build/cSpell.json @@ -34,6 +34,7 @@ "vsprofiler", "vstest", "Tailcall", + "testadapter", "toolchains", "unmanaged" ], diff --git a/build/smoke-tests/TestAdapterConsumer/ConsumedBenchmark.cs b/build/smoke-tests/TestAdapterConsumer/ConsumedBenchmark.cs new file mode 100644 index 0000000000..fc74f1e53e --- /dev/null +++ b/build/smoke-tests/TestAdapterConsumer/ConsumedBenchmark.cs @@ -0,0 +1,22 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace TestAdapterConsumer; + +/// +/// A single benchmark, enough for the smoke test to check that the packaged adapter turns this project into a +/// Microsoft.Testing.Platform application that can list it. +/// +[Config(typeof(FastConfig))] +public class ConsumedBenchmark +{ + [Benchmark] + public int Add() => 1 + 1; + + private class FastConfig : ManualConfig + { + public FastConfig() => AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)); + } +} diff --git a/build/smoke-tests/TestAdapterConsumer/TestAdapterConsumer.csproj b/build/smoke-tests/TestAdapterConsumer/TestAdapterConsumer.csproj new file mode 100644 index 0000000000..f88d1a27f0 --- /dev/null +++ b/build/smoke-tests/TestAdapterConsumer/TestAdapterConsumer.csproj @@ -0,0 +1,26 @@ + + + + + + net10.0 + Exe + enable + enable + + $(MSBuildThisFileDirectory)..\..\..\artifacts + + + + + + + + diff --git a/build/smoke-tests/test-adapter-consumer.ps1 b/build/smoke-tests/test-adapter-consumer.ps1 new file mode 100644 index 0000000000..47c9e586a5 --- /dev/null +++ b/build/smoke-tests/test-adapter-consumer.ps1 @@ -0,0 +1,115 @@ +#!/usr/bin/env pwsh + +<# +.SYNOPSIS + Smoke tests the packed BenchmarkDotNet.TestAdapter against a project that consumes it as a NuGet package. + +.DESCRIPTION + Everything in this repository that uses the adapter's build files imports them by path from the project file, + which MSBuild evaluates before nuget.g.targets. A package consumer gets the opposite order: NuGet imports + Microsoft.Testing.Platform.MSBuild's targets, which default IsTestingPlatformApplication to true, and only then + the adapter's, which have to overwrite that default for the opt-outs to work. Nothing in the solution can + reproduce that order, so this restores the real package and asserts on how the properties resolve. + + Run `build.cmd pack` first, so that the packages exist. + +.PARAMETER ArtifactsDirectory + The directory `build.cmd pack` wrote the packages to. + +.PARAMETER Configuration + The configuration to build the consuming project in. +#> + +[CmdletBinding()] +param( + [string] $ArtifactsDirectory = [System.IO.Path]::Combine($PSScriptRoot, '..', '..', 'artifacts'), + [string] $Configuration = 'Release' +) + +$ErrorActionPreference = 'Stop' +$PSNativeCommandUseErrorActionPreference = $true + +$project = [System.IO.Path]::Combine($PSScriptRoot, 'TestAdapterConsumer', 'TestAdapterConsumer.csproj') +$targetFramework = 'net10.0' + +# $IsWindows only exists on PowerShell Core, where it is the only way to tell; Windows PowerShell is Windows by definition. +$onWindows = ($null -eq $IsWindows) -or $IsWindows + +# build.cmd installs the SDK the repository is pinned to into .dotnet, and only puts it on PATH for its own run. +$dotnet = [System.IO.Path]::Combine($PSScriptRoot, '..', '..', '.dotnet', $(if ($onWindows) { 'dotnet.exe' } else { 'dotnet' })) +if (-not (Test-Path $dotnet)) { + $dotnet = 'dotnet' +} + +$package = Get-ChildItem -Path $ArtifactsDirectory -Filter 'BenchmarkDotNet.TestAdapter.*.nupkg' | + Where-Object { $_.Name -notlike '*.symbols.nupkg' } | + Select-Object -First 1 + +if ($null -eq $package) { + throw "No BenchmarkDotNet.TestAdapter package was found in '$ArtifactsDirectory'. Run 'build.cmd pack' first." +} + +$version = $package.BaseName -replace '^BenchmarkDotNet\.TestAdapter\.', '' +Write-Output "Consuming BenchmarkDotNet.TestAdapter $version from $ArtifactsDirectory" + +function Invoke-Dotnet { + param([Parameter(ValueFromRemainingArguments = $true)] [string[]] $Arguments) + + $output = & $dotnet @Arguments 2>&1 | Out-String + + if ($LASTEXITCODE -ne 0) { + Write-Output $output + throw "'dotnet $($Arguments -join ' ')' failed with exit code $LASTEXITCODE." + } + + return $output +} + +function Assert-Property { + param( + [string] $Name, + [string] $Expected, + [string[]] $With = @() + ) + + $arguments = @($project, '-nologo', '-tl:off', "-p:BenchmarkDotNetVersion=$version", "-p:Configuration=$Configuration") + $With + @("-getProperty:$Name") + $actual = (Invoke-Dotnet msbuild @arguments).Trim() + + $description = if ($With.Count -eq 0) { 'by default' } else { "with $($With -join ' ')" } + + if ($actual -ne $Expected) { + throw "Expected $Name to be '$Expected' $description, but it was '$actual'." + } + + Write-Output " OK: $Name is '$Expected' $description" +} + +Write-Output '##[group]Restoring the consuming project' +Invoke-Dotnet restore $project "-p:BenchmarkDotNetVersion=$version" '-tl:off' | Write-Output +Write-Output '##[endgroup]' + +Write-Output 'Checking how the packaged build files resolve the test platform:' + +# Microsoft.Testing.Platform is the default, and the adapter leaves the entry point to it. +Assert-Property -Name 'IsTestingPlatformApplication' -Expected 'true' +Assert-Property -Name 'GenerateProgramFile' -Expected 'false' + +# The two opt-outs have to win over the default Microsoft.Testing.Platform.MSBuild sets in its own targets, which a +# package consumer imports before the adapter's. +Assert-Property -Name 'IsTestingPlatformApplication' -Expected 'false' -With '-p:BenchmarkDotNetUseVSTest=true' +Assert-Property -Name 'IsTestingPlatformApplication' -Expected 'false' -With '-p:GenerateProgramFile=false' + +Write-Output '##[group]Building the consuming project' +Invoke-Dotnet build $project '--no-restore' '-c' $Configuration "-p:BenchmarkDotNetVersion=$version" '-tl:off' | Write-Output +Write-Output '##[endgroup]' + +Write-Output 'Listing the benchmarks through the entry point Microsoft.Testing.Platform generated:' +$application = [System.IO.Path]::Combine($PSScriptRoot, 'TestAdapterConsumer', 'bin', $Configuration, $targetFramework, 'TestAdapterConsumer.dll') +$listed = Invoke-Dotnet $application '--list-tests' '--no-ansi' +Write-Output $listed + +if ($listed -notmatch 'TestAdapterConsumer\.ConsumedBenchmark\.Add') { + throw 'The packaged adapter did not list the benchmark of the consuming project.' +} + +Write-Output 'The packaged BenchmarkDotNet.TestAdapter behaves as expected.' diff --git a/docs/articles/features/testadapter.md b/docs/articles/features/testadapter.md new file mode 100644 index 0000000000..8a091886fb --- /dev/null +++ b/docs/articles/features/testadapter.md @@ -0,0 +1,209 @@ +--- +uid: docs.testadapter +name: Running benchmarks as tests +--- + +# Running benchmarks as tests + +`BenchmarkDotNet.TestAdapter` lets your IDE and `dotnet test` discover and execute your benchmarks the way they do + unit tests. +This provides an alternative user experience to running benchmarks with the CLI + and may be preferable for those who like their IDE's test integrations that they may have used when running unit tests. + +Below is an example of running some benchmarks from the BenchmarkDotNet samples project in Visual Studio's Test Explorer. + +![](../../images/vs-testexplorer-demo.png) + +The adapter supports two test platforms: + +* [Microsoft.Testing.Platform](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-intro) (MTP), + the platform that `dotnet test` and modern IDE integrations are moving to. **This is the default.** +* [VSTest](xref:docs.vstest), for tooling without Microsoft.Testing.Platform support (such as Visual Studio 2019) + and for solutions that mix benchmark projects with VSTest based test projects. + +The difference that matters most is *where your benchmarks run*: + +* With **VSTest**, an external `testhost` process loads your benchmark assembly and the adapter reflects into it. +* With **Microsoft.Testing.Platform**, your benchmark project *is* the test host. + There is no separate host process, so BenchmarkDotNet behaves exactly as it does when you run the app from the CLI, + and the adapter does not need the child `AppDomain` that the VSTest adapter uses to load your assemblies correctly. + +## Caveats and things to know + +* **The benchmark measurements may be affected by the test host and your IDE!** + If you want accurate measurements, + it is still recommended to run benchmarks through the CLI without other processes impacting performance. + The measurements remain useful during development when comparing different approaches. +* **The adapter will not display or execute benchmarks if optimizations are disabled.** + Please ensure you are compiling in Release mode or with `Optimize` set to true. + Using an `InProcess` toolchain will let you run your benchmarks with optimizations disabled + and will let you attach the debugger as well. +* **The adapter will not call your application's entry point.** + If you use the entry point to customize how your benchmarks are run, + you will need to do this through other means such as an assembly-level `IConfigSource`, + as shown in [Setting a default configuration](xref:docs.vstest#setting-a-default-configuration). +* **The adapter will generate an entry point for you automatically.** + The generated entry point starts the test application. + See [Keeping your own entry point](#keeping-your-own-entry-point) if your project already has one. + +## Getting started + +* **Step 1.** Install the NuGet package. + Only one package is needed; it brings in `Microsoft.Testing.Platform` and the MSBuild integration for you: + +```xml + + + +``` + +* **Step 2.** Make sure the project is an executable and does not define its own entry point. + Microsoft.Testing.Platform applications are executables, and the package generates the entry point for you. + Here is a complete `.csproj` based on the default Console Application template: + +```xml + + + + Exe + net10.0 + enable + enable + + + + + + + +``` + +> [!NOTE] +> The name of your project file must match the name of the produced assembly. +> This is a general BenchmarkDotNet requirement: it rebuilds your project to run benchmarks out of process. + +* **Step 3.** Opt into the Microsoft.Testing.Platform mode of `dotnet test`. + On the .NET 10 SDK and later this is required, because `dotnet test` runs in VSTest mode by default. + Add a `global.json` next to your solution: + +```json +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} +``` + + On the .NET 9 SDK and earlier this step is not needed: + the package sets `TestingPlatformDotnetTestSupport` for you, which routes `dotnet test` to the platform. + +* **Step 4.** Switch to the `Release` configuration. + As mentioned above, the adapter does not discover or run benchmarks with optimizations disabled (by design). + +* **Step 5.** Build and run. + +```console +dotnet test -c Release +``` + + You can also run the produced executable directly, which is the same thing without going through MSBuild: + +```console +dotnet run -c Release +``` + +If this doesn't work for you, don't hesitate to file [a new GitHub issue](https://github.com/dotnet/BenchmarkDotNet/issues/new). + +## Listing and filtering benchmarks + +The benchmark project is a normal Microsoft.Testing.Platform application, so it accepts the platform's options. +Run it with `--help` to see all of them; the ones you are most likely to want are: + +```console +# List the benchmarks without running them. +dotnet run -c Release -- --list-tests + +# Run every benchmark of a class. +dotnet run -c Release -- --treenode-filter "/*/*/MyBenchmarks/*" + +# Run every benchmark of a category. +dotnet run -c Release -- --treenode-filter "/*/*/*/*[Category=Fast]" +``` + +The tree node filter path is `////`, + and `[BenchmarkCategory]` attributes are exposed as a `Category` trait that the filter can match on. +Because the platform separates the levels of that path with `/`, a benchmark parameter whose value contains one is + percent encoded in the path: a parameter value of `a/b` is written `a%2Fb` in a filter, and a literal `%` is + written `%25`. +This affects the filter only; the name the benchmark is displayed under is unchanged. + +## Keeping your own entry point + +The generated entry point starts the test application, which means it replaces the `BenchmarkSwitcher` entry point that + a benchmark project normally has. +There are two ways to keep your own. + +To keep a plain `BenchmarkSwitcher` entry point, tell the adapter that the project generates its own: + +```xml + + + false + +``` + +The project is then a normal console application again, and no test platform integration is set up for it. + +To keep an entry point *and* the test integration, start the test application yourself: + +```xml + + true + false + +``` + +```csharp +using BenchmarkDotNet.TestAdapter.TestingPlatform; +using Microsoft.Testing.Platform.Builder; + +public static class Program +{ + public static async Task Main(string[] args) + { + var builder = await TestApplication.CreateBuilderAsync(args); + builder.AddBenchmarkDotNet(); + using var app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} +``` + +From there you are free to decide when to start the test application and when to hand over to `BenchmarkSwitcher`, + for example by looking at the arguments your CI passes. + +## Using VSTest instead + +Set `BenchmarkDotNetUseVSTest` and add the VSTest host package: + +```xml + + true + + + + + + +``` + +See [Running with VSTest](xref:docs.vstest) for the details, including the IDE settings that VSTest integration needs. + +## Viewing the results + +The full BenchmarkDotNet output, including the summary table that compares benchmarks with each other, + is written to the test run output. + +In addition, each individual benchmark reports its own output, containing a histogram and various statistics for that + single benchmark case. +Depending on your IDE, this is shown when selecting the test after running it. diff --git a/docs/articles/features/toc.yml b/docs/articles/features/toc.yml index b456ed66be..db72e5c3e2 100644 --- a/docs/articles/features/toc.yml +++ b/docs/articles/features/toc.yml @@ -16,5 +16,7 @@ href: event-pipe-profiler.md - name: VSProfiler href: vsprofiler.md +- name: Benchmarks as tests + href: testadapter.md - name: VSTest href: vstest.md \ No newline at end of file diff --git a/docs/articles/features/vstest.md b/docs/articles/features/vstest.md index 3e94901bdd..2e0b3368d3 100644 --- a/docs/articles/features/vstest.md +++ b/docs/articles/features/vstest.md @@ -5,6 +5,11 @@ name: Running with VSTest # Running with VSTest +> [!NOTE] +> `BenchmarkDotNet.TestAdapter` runs your benchmarks through +> [Microsoft.Testing.Platform](xref:docs.testadapter) by default, which is the platform that succeeds VSTest. +> VSTest is opt-in, as described below. + BenchmarkDotNet supports discovering and executing benchmarks through VSTest. This provides an alternative user experience to running benchmarks with the CLI and may be preferable for those who like their IDE's VSTest integrations that they may have used when running unit tests. @@ -54,7 +59,17 @@ In addition, we can still make use of this boolean output to indicate You need to install two packages into your benchmark project: * `BenchmarkDotNet.TestAdapter`: Implements the VSTest protocol for BenchmarkDotNet * `Microsoft.NET.Test.Sdk`: Includes all the pieces needed for the VSTest host to run and load the VSTest adapter. -* **Step 2.** Make sure that the entry point is configured correctly. +* **Step 2.** Ask the adapter for VSTest. + `BenchmarkDotNet.TestAdapter` uses [Microsoft.Testing.Platform](xref:docs.testadapter) unless you set + `BenchmarkDotNetUseVSTest` in your project file: + +```xml + + true + +``` + +* **Step 3.** Make sure that the entry point is configured correctly. As mentioned in the caveats section, `BenchmarkDotNet.TestAdapter` will generate an entry point for you automatically. So, if you have an entry point already, you will either need to delete it or set `GenerateProgramFile` to `false` in your project file to continue using your existing one. @@ -68,6 +83,8 @@ In addition, we can still make use of this boolean output to indicate net8.0 enable enable + + true false @@ -80,7 +97,7 @@ In addition, we can still make use of this boolean output to indicate ``` -* **Step 3.** Make sure that your IDE supports VSTest integration. +* **Step 4.** Make sure that your IDE supports VSTest integration. In Visual Studio, everything works out of the box. In Rider/R#, the VSTest integration might need to be activated: * Go to the "Unit Testing" settings page. @@ -88,9 +105,9 @@ In addition, we can still make use of this boolean output to indicate * R#: Extensions -> ReSharper -> Options -> Tools -> Unit Testing -> Test Frameworks -> VSTest * Make sure that the "Enable VSTest adapter support" checkbox is checked. In recent versions of Rider, this may be enabled by default. -* **Step 4.** Switch to the `Release` configuration. +* **Step 5.** Switch to the `Release` configuration. As mentioned above, the TestAdapter is not able to discover and run benchmarks with optimizations disabled (by design). -* **Step 5.** Build the project. +* **Step 6.** Build the project. In order to discover the benchmarks, the VSTest adapter needs to be able to find the assembly. Once you build the project, you should observe the discovered benchmarks in your IDE's Unit Test Explorer. diff --git a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj index eeee8b90f2..fd51c0e8bf 100644 --- a/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj +++ b/samples/BenchmarkDotNet.Samples.FSharp/BenchmarkDotNet.Samples.FSharp.fsproj @@ -25,5 +25,13 @@ + + + + diff --git a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj index 4aa0c95e8c..ffa2cd63d3 100644 --- a/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj +++ b/samples/BenchmarkDotNet.Samples/BenchmarkDotNet.Samples.csproj @@ -38,5 +38,13 @@ + + + + diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs index 06e5782b6a..08ad640e26 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkCaseExtensions.cs @@ -1,5 +1,4 @@ using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; @@ -65,26 +64,6 @@ internal static TestCase ToVsTestCase(this BenchmarkCase benchmarkCase, string a return vsTestCase; } - /// - /// If an ID is not provided, a random string is used for the ID. This method will identify if randomness was - /// used for the ID and return the Job's DisplayInfo with that randomness removed so that the same benchmark - /// can be referenced across multiple processes. - /// - /// The benchmark case. - /// The benchmark case' job's DisplayInfo without randomness. - internal static string GetUnrandomizedJobDisplayInfo(this BenchmarkCase benchmarkCase) - { - var jobDisplayInfo = benchmarkCase.Job.DisplayInfo; - if (!benchmarkCase.Job.HasValue(CharacteristicObject.IdCharacteristic) && - benchmarkCase.Job.ResolvedId.StartsWith("Job-", StringComparison.OrdinalIgnoreCase)) - { - // Replace Job-ABCDEF with Job - jobDisplayInfo = "Job" + jobDisplayInfo.Substring(benchmarkCase.Job.ResolvedId.Length); - } - - return jobDisplayInfo; - } - /// /// Gets an ID for a given BenchmarkCase that is uniquely identifiable from discovery to execution phase. /// diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj index 3deb70b1cd..7cdb59410a 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkDotNet.TestAdapter.csproj @@ -5,8 +5,16 @@ BenchmarkDotNet.TestAdapter BenchmarkDotNet.TestAdapter BenchmarkDotNet.TestAdapter + Runs BenchmarkDotNet benchmarks as tests, through Microsoft.Testing.Platform or VSTest README.md True + + + false @@ -20,6 +28,13 @@ + + + + + @@ -28,7 +43,10 @@ - + + + diff --git a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs index e8cbc4f130..f001ef45f8 100644 --- a/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs +++ b/src/BenchmarkDotNet.TestAdapter/BenchmarkEnumerator.cs @@ -48,8 +48,16 @@ public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPa }; #endif - var assembly = Assembly.LoadFrom(assemblyPath); + return GetBenchmarksFromAssembly(Assembly.LoadFrom(assemblyPath)); + } + /// + /// Returns all the BenchmarkRunInfo objects from an already loaded assembly. + /// + /// The assembly of the benchmark project. + /// The benchmarks inside the assembly. + public static BenchmarkRunInfo[] GetBenchmarksFromAssembly(Assembly assembly) + { var isDebugAssembly = assembly.IsJitOptimizationDisabled() ?? false; return GenericBenchmarksBuilder.GetRunnableBenchmarks(assembly.GetRunnableBenchmarks()) @@ -59,7 +67,7 @@ public static BenchmarkRunInfo[] GetBenchmarksFromAssemblyPath(string assemblyPa if (isDebugAssembly) { // If the assembly is a debug assembly, then only display them if they will run in-process - // This will allow people to debug their benchmarks using VSTest if they wish. + // This will allow people to debug their benchmarks from a test runner if they wish. benchmarkRunInfo = new BenchmarkRunInfo( benchmarkRunInfo.BenchmarksCases.Where(c => c.GetToolchain().IsInProcess).ToArray(), benchmarkRunInfo.Type, diff --git a/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkDotNetExtension.cs b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkDotNetExtension.cs new file mode 100644 index 0000000000..6982414812 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkDotNetExtension.cs @@ -0,0 +1,34 @@ +using Microsoft.Testing.Platform.Extensions; + +namespace BenchmarkDotNet.TestAdapter.TestingPlatform +{ + /// + /// Identifies BenchmarkDotNet to Microsoft.Testing.Platform. + /// + /// + /// Several platform services, such as the tree node filter behind --treenode-filter, are registered on behalf of an + /// extension rather than of the test framework itself, so the identity lives in its own type. + /// + internal sealed class BenchmarkDotNetExtension : IExtension + { + /// + /// The uid shared by every extension this package registers. + /// + public const string ExtensionUid = "BenchmarkDotNet.TestAdapter"; + + /// + public string Uid => ExtensionUid; + + /// + public string Version => typeof(BenchmarkDotNetExtension).Assembly.GetName().Version?.ToString() ?? "0.0.0"; + + /// + public string DisplayName => "BenchmarkDotNet"; + + /// + public string Description => "Runs BenchmarkDotNet benchmarks as tests."; + + /// + public Task IsEnabledAsync() => Task.FromResult(true); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkEventProcessor.cs b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkEventProcessor.cs new file mode 100644 index 0000000000..70b0839469 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkEventProcessor.cs @@ -0,0 +1,201 @@ +using BenchmarkDotNet.EventProcessors; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Results; +using BenchmarkDotNet.Validators; +using Microsoft.Testing.Platform.Extensions.Messages; +using Perfolizer.Mathematics.Histograms; +using System.Diagnostics; +using System.Globalization; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter.TestingPlatform +{ + + /// + /// Translates the BenchmarkDotNet run into the stream of test node updates the platform expects. + /// + internal sealed class BenchmarkEventProcessor : EventProcessor + { + private readonly IReadOnlyDictionary nodes; + private readonly Action publish; + private readonly Stopwatch runTimerStopwatch = new(); + private readonly Dictionary pendingResults = []; + private readonly HashSet publishedResults = []; + + // BenchmarkDotNet builds the partitions in parallel and raises OnBuildComplete from each of the build tasks, + // so that callback can run on several threads at once. The others cannot: every build is awaited before the + // first benchmark runs, and the benchmarks themselves run one after another. + private readonly object buildCompleteGate = new(); + + public BenchmarkEventProcessor(IReadOnlyDictionary nodes, Action publish) + { + this.nodes = nodes; + this.publish = publish; + } + + public override void OnValidationError(ValidationError validationError) + { + // If the error is not linked to a benchmark case, then set the error on all benchmarks. + var affected = validationError.BenchmarkCase == null + ? nodes.Values + : [nodes[validationError.BenchmarkCase.GetUniqueId()]]; + + foreach (var node in affected) + { + var pending = GetOrCreatePendingResult(node); + + if (validationError.IsCritical) + { + // The result is not published yet, in case there are more validation errors to append. + pending.ErrorMessages.Add(validationError.Message); + } + else + { + pending.Output.AppendLine($"WARNING: {validationError.Message}"); + } + } + } + + public override void OnBuildComplete(BuildPartition buildPartition, BuildResult buildResult) + { + // Only build failures need to be reported, successful builds are followed by a run. + if (buildResult.IsBuildSuccess) + return; + + lock (buildCompleteGate) + { + foreach (var benchmarkBuildInfo in buildPartition.Benchmarks) + { + var node = nodes[benchmarkBuildInfo.BenchmarkCase.GetUniqueId()]; + var pending = GetOrCreatePendingResult(node); + + if (buildResult.GenerateException != null) + pending.ErrorMessages.Add($"// Generate Exception: {buildResult.GenerateException.Message}"); + else if (buildResult.TryToExplainFailureReason(buildPartition.GetInProcessDiagnoserHandlerTypes(), out string? reason)) + pending.ErrorMessages.Add($"// Build Error: {reason}"); + else if (buildResult.ErrorMessage != null) + pending.ErrorMessages.Add($"// Build Error: {buildResult.ErrorMessage}"); + + // A benchmark that failed to build will never run, so the result can be published immediately. + publish(node.ToTestNode(InProgressTestNodeStateProperty.CachedInstance)); + PublishResult(node, pending, new FailedTestNodeStateProperty(pending.GetErrorMessage() ?? "The benchmark failed to build.")); + } + } + } + + public override void OnStartRunBenchmark(BenchmarkCase benchmarkCase) + { + var node = nodes[benchmarkCase.GetUniqueId()]; + var pending = GetOrCreatePendingResult(node); + pending.StartTime = DateTimeOffset.UtcNow; + + publish(node.ToTestNode(InProgressTestNodeStateProperty.CachedInstance)); + runTimerStopwatch.Restart(); + } + + public override void OnEndRunBenchmark(BenchmarkCase benchmarkCase, BenchmarkReport report) + { + var node = nodes[benchmarkCase.GetUniqueId()]; + var pending = GetOrCreatePendingResult(node); + pending.Duration = runTimerStopwatch.Elapsed; + pending.EndTime = DateTimeOffset.UtcNow; + + AppendMeasurementSummary(pending.Output, report); + + TestNodeStateProperty state = report.Success && pending.ErrorMessages.Count == 0 + ? PassedTestNodeStateProperty.CachedInstance + : new FailedTestNodeStateProperty(pending.GetErrorMessage() ?? "The benchmark did not complete successfully."); + + PublishResult(node, pending, state); + } + + /// + /// Publishes a result for every benchmark that was scheduled to run but never reported one, which happens when + /// a critical validation error stopped it or when BenchmarkDotNet never reached it. + /// + public void PublishOutstandingResults() + { + foreach (var node in nodes.Values) + { + if (publishedResults.Contains(node.Uid)) + continue; + + var pending = GetOrCreatePendingResult(node); + var errorMessage = pending.GetErrorMessage(); + + TestNodeStateProperty state = errorMessage != null + ? new FailedTestNodeStateProperty(errorMessage) + : SkippedTestNodeStateProperty.CachedInstance; + + publish(node.ToTestNode(InProgressTestNodeStateProperty.CachedInstance)); + PublishResult(node, pending, state); + } + } + + private void PublishResult(BenchmarkTestNode node, PendingResult pending, TestNodeStateProperty state) + { + var properties = new List(); + + if (pending.StartTime is { } startTime) + { + var duration = pending.Duration ?? TimeSpan.Zero; + properties.Add(new TimingProperty(new TimingInfo(startTime, pending.EndTime ?? startTime + duration, duration))); + } + + if (pending.Output.Length > 0) + properties.Add(new StandardOutputProperty(pending.Output.ToString())); + + publish(node.ToTestNode(state, properties.ToArray())); + publishedResults.Add(node.Uid); + } + + private PendingResult GetOrCreatePendingResult(BenchmarkTestNode node) + { + if (!pendingResults.TryGetValue(node.Uid, out var pending)) + { + pending = new PendingResult(); + pendingResults[node.Uid] = pending; + } + + return pending; + } + + private static void AppendMeasurementSummary(StringBuilder output, BenchmarkReport report) + { + var resultRuns = report.GetResultRuns(); + if (resultRuns.Count == 0) + return; + + output.AppendLine(report.BenchmarkCase.DisplayInfo); + output.AppendLine($"Runtime = {report.GetRuntimeInfo()}; GC = {report.GetGcInfo()}"); + + var statistics = resultRuns.GetStatistics(); + var cultureInfo = CultureInfo.InvariantCulture; + var formatter = statistics.CreateNanosecondFormatter(cultureInfo); + + var histogram = HistogramBuilder.Adaptive.Build(statistics.Sample.Values); + output.AppendLine("-------------------- Histogram --------------------"); + output.AppendLine(histogram.ToString(formatter)); + output.AppendLine("---------------------------------------------------"); + output.AppendLine(statistics.ToString(cultureInfo, formatter, calcHistogram: false)); + } + + private sealed class PendingResult + { + public List ErrorMessages { get; } = []; + + public StringBuilder Output { get; } = new(); + + public DateTimeOffset? StartTime { get; set; } + + public DateTimeOffset? EndTime { get; set; } + + public TimeSpan? Duration { get; set; } + + public string? GetErrorMessage() => ErrorMessages.Count == 0 ? null : string.Join("\n", ErrorMessages); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkTestFramework.cs b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkTestFramework.cs new file mode 100644 index 0000000000..bdce27cd9d --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkTestFramework.cs @@ -0,0 +1,319 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestFramework; +using Microsoft.Testing.Platform.Requests; +using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.TestHost; +using System.Reflection; +using System.Runtime.ExceptionServices; +using System.Threading.Channels; + +namespace BenchmarkDotNet.TestAdapter.TestingPlatform +{ + /// + /// Discovers and executes the benchmarks of the running assembly through Microsoft.Testing.Platform. + /// + internal sealed class BenchmarkTestFramework : ITestFramework, IDataProducer, IOutputDeviceDataProducer + { + private readonly BenchmarkDotNetExtension extension = new(); + private readonly IServiceProvider serviceProvider; + private readonly Assembly assembly; + + public BenchmarkTestFramework(ITestFrameworkCapabilities capabilities, IServiceProvider serviceProvider, Assembly assembly) + { + Capabilities = capabilities; + this.serviceProvider = serviceProvider; + this.assembly = assembly; + } + + /// + public string Uid => extension.Uid; + + /// + public string Version => extension.Version; + + /// + public string DisplayName => extension.DisplayName; + + /// + public string Description => extension.Description; + + /// + public Type[] DataTypesProduced => [typeof(TestNodeUpdateMessage)]; + + /// + /// Gets the capabilities the framework was registered with. + /// + public ITestFrameworkCapabilities Capabilities { get; } + + /// + public Task IsEnabledAsync() => extension.IsEnabledAsync(); + + /// + public Task CreateTestSessionAsync(CreateTestSessionContext context) + => Task.FromResult(new CreateTestSessionResult { IsSuccess = true }); + + /// + public Task CloseTestSessionAsync(CloseTestSessionContext context) + => Task.FromResult(new CloseTestSessionResult { IsSuccess = true }); + + /// + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + try + { + switch (context.Request) + { + case DiscoverTestExecutionRequest discoverRequest: + await DiscoverAsync(discoverRequest, context).ConfigureAwait(false); + break; + case RunTestExecutionRequest runRequest: + await RunAsync(runRequest, context).ConfigureAwait(false); + break; + } + } + finally + { + context.Complete(); + } + } + + private async Task DiscoverAsync(DiscoverTestExecutionRequest request, ExecuteRequestContext context) + { + foreach (var benchmarks in GetMatchingBenchmarks(request.Filter)) + { + context.CancellationToken.ThrowIfCancellationRequested(); + + // Exactly one node per uid: publishing a colliding uid twice would leave the platform with two nodes + // it cannot tell apart. The collision itself is reported when the benchmarks are run. + var message = new TestNodeUpdateMessage( + request.Session.SessionUid, + benchmarks[0].Node.ToTestNode(DiscoveredTestNodeStateProperty.CachedInstance)); + + await context.MessageBus.PublishAsync(this, message).ConfigureAwait(false); + } + } + + private async Task RunAsync(RunTestExecutionRequest request, ExecuteRequestContext context) + { + var sessionUid = request.Session.SessionUid; + var cancellationToken = context.CancellationToken; + + var runnable = new List(); + foreach (var benchmarks in GetMatchingBenchmarks(request.Filter)) + { + if (benchmarks.Count == 1) + runnable.Add(benchmarks[0]); + else + await PublishCollisionAsync(context, sessionUid, benchmarks).ConfigureAwait(false); + } + + if (runnable.Count == 0) + return; + + var nodes = runnable.ToDictionary(match => match.Node.Uid, match => match.Node); + + // BenchmarkDotNet reports its progress through synchronous callbacks (EventProcessor and ILogger) while + // the message bus and the output device are asynchronous. Blocking on those from inside a callback risks + // deadlocking against the synchronization context BenchmarkDotNet installs while it runs, so the callbacks + // write to this channel and the drain below does the awaiting. Synchronous continuations are left off, so + // that a write can never end up publishing on BenchmarkDotNet's own thread. + var workQueue = Channel.CreateUnbounded>(new UnboundedChannelOptions + { + SingleReader = true, + AllowSynchronousContinuations = false + }); + + // A failure while publishing has to stop the benchmarks as well, otherwise the run would carry on with + // nobody listening to it. + using var runCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + + var eventProcessor = new BenchmarkEventProcessor(nodes, testNode => + { + var message = new TestNodeUpdateMessage(sessionUid, testNode); + workQueue.Writer.TryWrite(() => context.MessageBus.PublishAsync(this, message)); + }); + + // BenchmarkDotNet's own console output is replaced so that everything goes through the output device, + // which keeps it in the right place when the platform runs in server mode or inside an IDE. + var logger = new OutputDeviceLogger(serviceProvider.GetOutputDevice(), this, workQueue.Writer, cancellationToken); + + var runInfos = runnable + .GroupBy(match => match.RunInfo) + .Select(group => new BenchmarkRunInfo( + group.Select(match => match.Node.BenchmarkCase).ToArray(), + group.Key.Type, + group.Key.Config + .AddEventProcessor(eventProcessor) + .AddLogger(logger) + .RemoveLoggersOfType() + .CreateImmutableConfig(), + group.Key.CompositeInProcessDiagnoser)) + .ToArray(); + + // BenchmarkDotNet blocks the calling thread for the whole run, so it gets a thread of its own and the + // queued messages are published from here as they are produced. + var runTask = Task.Run( + () => + { + try + { + BenchmarkRunner.Run(runInfos, runCancellation.Token); + } + finally + { + try + { + // Benchmarks that never reported a result still need one, unless the run was cancelled: + // the platform expects an OperationCanceledException in that case, and publishing results + // afterwards would contradict it. + if (!runCancellation.IsCancellationRequested) + eventProcessor.PublishOutstandingResults(); + + logger.Flush(); + } + finally + { + // The drain only ends once the queue is completed, so this has to happen no matter what + // else went wrong. + workQueue.Writer.TryComplete(); + } + } + }, + CancellationToken.None); + + ExceptionDispatchInfo? drainFailure = null; + try + { + await DrainAsync(workQueue.Reader).ConfigureAwait(false); + } + catch (Exception exception) + { + // Nothing consumes the queue anymore, so the run has to be stopped rather than left orphaned. This is + // also the path a cancelled run takes, since the queued writes are handed the platform's token. + drainFailure = ExceptionDispatchInfo.Capture(exception); + runCancellation.Cancel(); + } + + try + { + // The run has to be over before the request completes, otherwise it would carry on in the background + // and its failure would go unobserved. + await runTask.ConfigureAwait(false); + } + catch when (drainFailure != null) + { + // The run was stopped because publishing failed, so that failure is the one worth reporting. + } + + drainFailure?.Throw(); + } + + /// + /// Runs the queued work items in order, until the queue is completed and empty. + /// + /// The reader of the work queue. + private static async Task DrainAsync(ChannelReader> reader) + { + while (await reader.WaitToReadAsync().ConfigureAwait(false)) + { + while (reader.TryRead(out var work)) + await work().ConfigureAwait(false); + } + } + + /// + /// Reports benchmarks that share a uid as a single failed test. + /// + /// + /// The platform identifies test nodes by uid, so benchmarks that produce the same one cannot be reported + /// separately. Failing them keeps the rest of the run going, which is more useful than aborting the request. + /// + private async Task PublishCollisionAsync(ExecuteRequestContext context, SessionUid sessionUid, List collision) + { + var node = collision[0].Node; + var error = + $"{collision.Count} benchmarks are identified as '{node.Uid}', so they cannot be told apart and none " + + "of them were run. Benchmarks are identified by the string representation of their parameters: give " + + "the colliding values distinct ToString() results."; + + await context.MessageBus.PublishAsync( + this, + new TestNodeUpdateMessage(sessionUid, node.ToTestNode(InProgressTestNodeStateProperty.CachedInstance))).ConfigureAwait(false); + + await context.MessageBus.PublishAsync( + this, + new TestNodeUpdateMessage(sessionUid, node.ToTestNode(new FailedTestNodeStateProperty(error)))).ConfigureAwait(false); + } + + /// + /// Enumerates the benchmarks of the assembly and keeps the ones the request asked for. + /// + /// The filter of the request. + /// + /// The matching benchmarks in enumeration order, grouped by uid. A group holding more than one benchmark is a + /// uid collision. + /// + private List> GetMatchingBenchmarks(ITestExecutionFilter filter) + { + var matches = new List>(); + var matchesByUid = new Dictionary>(StringComparer.Ordinal); + + foreach (var runInfo in BenchmarkEnumerator.GetBenchmarksFromAssembly(assembly)) + { + // The job only earns a place in the display name when the benchmark actually runs under several jobs. + // This is computed before filtering so that a benchmark keeps the same name however it was selected. + var includeJobInName = runInfo.BenchmarksCases.Select(c => c.Job.DisplayInfo).Distinct().Count() > 1; + + foreach (var benchmarkCase in runInfo.BenchmarksCases) + { + var node = BenchmarkTestNode.Create(benchmarkCase, includeJobInName); + if (!Matches(filter, node)) + continue; + + if (!matchesByUid.TryGetValue(node.Uid, out var sameUid)) + { + sameUid = new List(); + matchesByUid.Add(node.Uid, sameUid); + matches.Add(sameUid); + } + + sameUid.Add(new Match(runInfo, node)); + } + } + + return matches; + } + +#pragma warning disable TPEXP // The tree node filter is still marked as experimental by the platform. + private static bool Matches(ITestExecutionFilter filter, BenchmarkTestNode node) => filter switch + { + TestNodeUidListFilter uidListFilter => uidListFilter.TestNodeUids.Any(uid => uid.Value == node.Uid), + TreeNodeFilter treeNodeFilter => treeNodeFilter.MatchesFilter(node.Path, node.GetFilterableProperties()), + + // NopFilter, and anything the platform adds later, means "everything". + _ => true + }; +#pragma warning restore TPEXP + + /// + /// A benchmark that matched the request, together with the run info it belongs to. + /// + private sealed class Match + { + public Match(BenchmarkRunInfo runInfo, BenchmarkTestNode node) + { + RunInfo = runInfo; + Node = node; + } + + public BenchmarkRunInfo RunInfo { get; } + + public BenchmarkTestNode Node { get; } + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkTestNode.cs b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkTestNode.cs new file mode 100644 index 0000000000..9d79bd7f35 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/BenchmarkTestNode.cs @@ -0,0 +1,160 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Exporters; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; +using Microsoft.Testing.Platform.Extensions.Messages; +using System.Reflection; +using System.Text; + +namespace BenchmarkDotNet.TestAdapter.TestingPlatform +{ + /// + /// The Microsoft.Testing.Platform view of a single . + /// + /// + /// A carries mutable state (the property bag holds the current outcome), so a fresh node is + /// created for every message published on the bus. This class holds the parts that never change. + /// + internal sealed class BenchmarkTestNode + { + private readonly IProperty[] staticProperties; + + private BenchmarkTestNode(BenchmarkCase benchmarkCase, string uid, string displayName, string path, IProperty[] staticProperties) + { + BenchmarkCase = benchmarkCase; + Uid = uid; + DisplayName = displayName; + Path = path; + this.staticProperties = staticProperties; + } + + /// + /// Gets the benchmark this node represents. + /// + public BenchmarkCase BenchmarkCase { get; } + + /// + /// Gets the stable identifier of the node. It has to be identical in the discovery and the execution phase, + /// which may happen in different processes. + /// + public string Uid { get; } + + /// + /// Gets the name shown by test runners. + /// + public string DisplayName { get; } + + /// + /// Gets the '/' separated path used by . + /// + public string Path { get; } + + /// + /// Creates the node for a benchmark case. + /// + /// The benchmark case to describe. + /// + /// Whether the display name should be suffixed with the job, which is only useful when the benchmark runs + /// under more than one job. + /// + /// The created node. + public static BenchmarkTestNode Create(BenchmarkCase benchmarkCase, bool includeJobInName) + { + var benchmarkMethod = benchmarkCase.Descriptor.WorkloadMethod; + var type = benchmarkCase.Descriptor.Type; + var fullClassName = type.GetCorrectCSharpTypeName(prefixWithGlobal: false); + var parametrizedMethodName = FullNameProvider.GetMethodName(benchmarkCase); + var jobDisplayInfo = benchmarkCase.GetUnrandomizedJobDisplayInfo(); + + // The uid is the hash BenchmarkDotNet itself uses (and reports through `--list json`), so that a benchmark + // keeps the same identity across processes and across tools. The job is only part of the display name + // when it actually adds information. + var uid = benchmarkCase.GetUniqueId(); + + // Microsoft.Testing.Platform keeps the display name and the identity apart, so the name is free to be the + // [Benchmark(Description = ...)] the author chose. GetMethodDisplayName falls back to the method name when + // no description is set. The path keeps the method name, so that a filter still matches what + // BenchmarkDotNet's own --filter matches. + var displayMethodName = FullNameProvider.GetMethodDisplayName(benchmarkCase); + var displayName = $"{fullClassName}.{displayMethodName}" + (includeJobInName ? $" [{jobDisplayInfo}]" : ""); + + var properties = new List + { + new TestMethodIdentifierProperty( + type.Assembly.FullName, + type.Namespace ?? string.Empty, + type.GetCorrectCSharpTypeName(prefixWithGlobal: false, includeNamespace: false), + benchmarkMethod.Name, + benchmarkMethod.IsGenericMethodDefinition ? benchmarkMethod.GetGenericArguments().Length : 0, + benchmarkMethod.GetParameters().Select(p => p.ParameterType.FullName ?? p.ParameterType.Name).ToArray(), + benchmarkMethod.ReturnType.FullName ?? benchmarkMethod.ReturnType.Name), + }; + + var benchmarkAttribute = benchmarkMethod.ResolveAttribute(); + if (benchmarkAttribute?.SourceCodeFile != null) + { + // BenchmarkAttribute captures the line of the attribute itself, and the platform expects 0-based lines. + var line = Math.Max(0, benchmarkAttribute.SourceCodeLineNumber - 1); + var position = new LinePosition(line, 0); + properties.Add(new TestFileLocationProperty(benchmarkAttribute.SourceCodeFile, new LinePositionSpan(position, position))); + } + + foreach (var category in DefaultCategoryDiscoverer.Instance.GetCategories(benchmarkMethod)) + properties.Add(new TestMetadataProperty("Category", category)); + + var path = BuildPath(type.Assembly, type.Namespace, fullClassName, parametrizedMethodName, jobDisplayInfo); + + return new BenchmarkTestNode(benchmarkCase, uid, displayName, path, properties.ToArray()); + } + + /// + /// Creates a message-bus ready node in the given state. + /// + /// The state of the benchmark, e.g. discovered, passed or failed. + /// Any additional properties, such as timing or captured output. + /// The created test node. + public TestNode ToTestNode(TestNodeStateProperty state, params IProperty[] extraProperties) + { + var properties = new PropertyBag(staticProperties); + properties.Add(state); + foreach (var property in extraProperties) + properties.Add(property); + + return new TestNode + { + Uid = new TestNodeUid(Uid), + DisplayName = DisplayName, + Properties = properties + }; + } + + /// + /// Gets the properties a can match against, + /// which is what makes `--treenode-filter "/*/*/*/*[Category=Fast]"` work. + /// + /// The filterable properties. + public PropertyBag GetFilterableProperties() => new PropertyBag(staticProperties); + + private static string BuildPath(Assembly assembly, string? @namespace, string fullClassName, string methodName, string jobDisplayInfo) + { + // The convention followed by the other test frameworks is ////. + var className = @namespace == null || !fullClassName.StartsWith(@namespace + ".", StringComparison.Ordinal) + ? fullClassName + : fullClassName.Substring(@namespace.Length + 1); + + return new StringBuilder() + .Append('/').Append(Escape(assembly.GetName().Name)) + .Append('/').Append(Escape(@namespace ?? string.Empty)) + .Append('/').Append(Escape(className)) + .Append('/').Append(Escape($"{methodName} [{jobDisplayInfo}]")) + .ToString(); + } + + // Benchmark parameters are stringified user values, so they can contain the path separator. A '/' cannot be + // escaped into a segment: Microsoft.Testing.Platform splits the path on every '/' without ever unescaping it, + // and TreeNodeFilter rejects a filter whose segment contains one, so a raw '/' would both deepen the tree and + // leave the benchmark unmatchable. Percent encoding keeps the path four levels deep and the segment + // addressable, at the price of a filter having to spell the separator as '%2F'. + private static string Escape(string segment) => segment.Replace("%", "%25").Replace("/", "%2F"); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/TestingPlatform/OutputDeviceLogger.cs b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/OutputDeviceLogger.cs new file mode 100644 index 0000000000..a2d1b0d2b4 --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/OutputDeviceLogger.cs @@ -0,0 +1,78 @@ +using BenchmarkDotNet.Loggers; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.OutputDevice; +using System.Text; +using System.Threading.Channels; + +namespace BenchmarkDotNet.TestAdapter.TestingPlatform +{ + /// + /// Forwards the BenchmarkDotNet log to the platform output device, so that build progress and the result summary + /// show up in the test run output. + /// + internal sealed class OutputDeviceLogger : ILogger + { + private readonly IOutputDevice outputDevice; + private readonly IOutputDeviceDataProducer producer; + private readonly ChannelWriter> workQueue; + private readonly CancellationToken cancellationToken; + private readonly StringBuilder currentLine = new(); + private LogKind currentLineKind = LogKind.Default; + + public OutputDeviceLogger( + IOutputDevice outputDevice, + IOutputDeviceDataProducer producer, + ChannelWriter> workQueue, + CancellationToken cancellationToken) + { + this.outputDevice = outputDevice; + this.producer = producer; + this.workQueue = workQueue; + this.cancellationToken = cancellationToken; + } + + public string Id => nameof(OutputDeviceLogger); + + public int Priority => 0; + + public void Write(LogKind logKind, string text) + { + currentLine.Append(text); + + // Assume that if any part of the line is an error or a warning, the whole line is. + // The kind is reset when the line is flushed. + if (logKind == LogKind.Error || (logKind == LogKind.Warning && currentLineKind != LogKind.Error)) + currentLineKind = logKind; + } + + public void WriteLine() + { + var text = currentLine.ToString(); + var kind = currentLineKind; + + currentLine.Clear(); + currentLineKind = LogKind.Default; + + IOutputDeviceData data = kind switch + { + LogKind.Error => new ErrorMessageOutputDeviceData(text), + LogKind.Warning => new WarningMessageOutputDeviceData(text), + _ => new TextOutputDeviceData(text) + }; + + workQueue.TryWrite(() => outputDevice.DisplayAsync(producer, data, cancellationToken)); + } + + public void WriteLine(LogKind logKind, string text) + { + Write(logKind, text); + WriteLine(); + } + + public void Flush() + { + if (currentLine.Length > 0) + WriteLine(); + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/TestingPlatform/TestApplicationBuilderExtensions.cs b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/TestApplicationBuilderExtensions.cs new file mode 100644 index 0000000000..e1eecf053a --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/TestApplicationBuilderExtensions.cs @@ -0,0 +1,50 @@ +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Helpers; +using System.Reflection; + +namespace BenchmarkDotNet.TestAdapter.TestingPlatform +{ + /// + /// Extensions for registering BenchmarkDotNet with a Microsoft.Testing.Platform application. + /// + public static class TestApplicationBuilderExtensions + { + /// + /// Registers BenchmarkDotNet as the test framework, so that the benchmarks of the entry assembly are exposed + /// as tests. + /// + /// The builder of the test application. + /// The same builder, so that calls can be chained. + public static ITestApplicationBuilder AddBenchmarkDotNet(this ITestApplicationBuilder builder) + => builder.AddBenchmarkDotNet( + Assembly.GetEntryAssembly() ?? throw new InvalidOperationException( + "There is no entry assembly to look for benchmarks in. Use the overload that takes an assembly.")); + + /// + /// Registers BenchmarkDotNet as the test framework, so that the benchmarks of the given assembly are exposed + /// as tests. + /// + /// The builder of the test application. + /// The assembly to look for benchmarks in. + /// The same builder, so that calls can be chained. + public static ITestApplicationBuilder AddBenchmarkDotNet(this ITestApplicationBuilder builder, Assembly assembly) + { + if (builder == null) + throw new ArgumentNullException(nameof(builder)); + if (assembly == null) + throw new ArgumentNullException(nameof(assembly)); + + builder.RegisterTestFramework( + _ => new TestFrameworkCapabilities(), + (capabilities, serviceProvider) => new BenchmarkTestFramework(capabilities, serviceProvider, assembly)); + + // Opts into the tree node filter, which is what backs `--treenode-filter "/*/*/MyBenchmarks/*"`. +#pragma warning disable TPEXP // The tree node filter is still marked as experimental by the platform. + builder.AddTreeNodeFilterService(new BenchmarkDotNetExtension()); +#pragma warning restore TPEXP + + return builder; + } + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/TestingPlatform/TestingPlatformBuilderHook.cs b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/TestingPlatformBuilderHook.cs new file mode 100644 index 0000000000..bc7c7db03c --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/TestingPlatform/TestingPlatformBuilderHook.cs @@ -0,0 +1,26 @@ +using Microsoft.Testing.Platform.Builder; +using System.ComponentModel; + +namespace BenchmarkDotNet.TestAdapter.TestingPlatform +{ + /// + /// The hook Microsoft.Testing.Platform.MSBuild calls from the entry point it generates for the benchmark project. + /// + /// + /// This is wired up by the TestingPlatformBuilderHook item in BenchmarkDotNet.TestAdapter.targets. + /// It is public because the generated code lives in the benchmark assembly, but it is not meant to be called + /// directly; use + /// instead when writing an entry point by hand. + /// + [EditorBrowsable(EditorBrowsableState.Never)] + public static class TestingPlatformBuilderHook + { + /// + /// Registers BenchmarkDotNet with the test application being built. + /// + /// The builder of the test application. + /// The command line arguments of the process. Unused. + public static void AddExtensions(ITestApplicationBuilder builder, string[] arguments) + => builder.AddBenchmarkDotNet(); + } +} diff --git a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props index 7184842aba..5d1eabd4f6 100644 --- a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props +++ b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.props @@ -1,15 +1,19 @@ - + $(MSBuildThisFileDirectory)..\entrypoints\ false - - + diff --git a/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.targets b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.targets new file mode 100644 index 0000000000..09d71557cf --- /dev/null +++ b/src/BenchmarkDotNet.TestAdapter/build/BenchmarkDotNet.TestAdapter.targets @@ -0,0 +1,49 @@ + + + + + + false + true + + + true + + + false + + + + + + BenchmarkDotNet + BenchmarkDotNet.TestAdapter.TestingPlatform.TestingPlatformBuilderHook + + + diff --git a/src/BenchmarkDotNet/Exporters/FullNameProvider.cs b/src/BenchmarkDotNet/Exporters/FullNameProvider.cs index 19555908d2..5411f7243b 100644 --- a/src/BenchmarkDotNet/Exporters/FullNameProvider.cs +++ b/src/BenchmarkDotNet/Exporters/FullNameProvider.cs @@ -99,6 +99,22 @@ internal static string GetMethodName(BenchmarkCase benchmarkCase) return name.ToString(); } + /// + /// Gets the method name to show to a user, which is the [Benchmark(Description = ...)] when one is set and + /// the method name otherwise, followed by the parameters. + /// + /// The benchmark case. + /// The method name to display. + internal static string GetMethodDisplayName(BenchmarkCase benchmarkCase) + { + var name = new StringBuilder(benchmarkCase.Descriptor.WorkloadMethodDisplayInfo); + + if (benchmarkCase.HasParameters) + name.Append(GetBenchmarkParameters(benchmarkCase.Descriptor.WorkloadMethod, benchmarkCase.Parameters)); + + return name.ToString(); + } + private static string GetBenchmarkParameters(MethodInfo method, ParameterInstances benchmarkParameters) { var methodArguments = method.GetParameters(); diff --git a/src/BenchmarkDotNet/Extensions/BenchmarkCaseIdentityExtensions.cs b/src/BenchmarkDotNet/Extensions/BenchmarkCaseIdentityExtensions.cs new file mode 100644 index 0000000000..948ea1d7fe --- /dev/null +++ b/src/BenchmarkDotNet/Extensions/BenchmarkCaseIdentityExtensions.cs @@ -0,0 +1,32 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Running; + +namespace BenchmarkDotNet.Extensions +{ + /// + /// Helpers for deriving stable identities for a BenchmarkCase. + /// + internal static class BenchmarkCaseIdentityExtensions + { + /// + /// If an ID is not provided, a random string is used for the ID. This method will identify if randomness was + /// used for the ID and return the Job's DisplayInfo with that randomness removed so that the same benchmark + /// can be referenced across multiple processes. + /// + /// The benchmark case. + /// The benchmark case' job's DisplayInfo without randomness. + internal static string GetUnrandomizedJobDisplayInfo(this BenchmarkCase benchmarkCase) + { + var jobDisplayInfo = benchmarkCase.Job.DisplayInfo; + if (!benchmarkCase.Job.HasValue(CharacteristicObject.IdCharacteristic) && + benchmarkCase.Job.ResolvedId.StartsWith("Job-", StringComparison.OrdinalIgnoreCase)) + { + // Replace Job-ABCDEF with Job + jobDisplayInfo = "Job" + jobDisplayInfo.Substring(benchmarkCase.Job.ResolvedId.Length); + } + + return jobDisplayInfo; + } + } +} + diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures.csproj b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures.csproj new file mode 100644 index 0000000000..610c70c194 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures.csproj @@ -0,0 +1,32 @@ + + + + net10.0 + Exe + BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures + BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures + BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures + + + true + + + + + + + + + + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/BuildFailureProbe.cs b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/BuildFailureProbe.cs new file mode 100644 index 0000000000..3665f8592a --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/BuildFailureProbe.cs @@ -0,0 +1,50 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Parameters; +using BenchmarkDotNet.Toolchains.Results; + +namespace BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures +{ + /// + /// A benchmark whose build always fails, so that the adapter has to turn the build failure into a failed test. + /// The failure is faked by the toolchain rather than by uncompilable code, which keeps it quick and keeps the + /// error message the test asserts on under this file's control. + /// + [Config(typeof(FailingBuildConfig))] + public class BuildFailureProbe + { + internal const string ErrorMessage = "The build of this benchmark always fails, on purpose."; + + [Benchmark] + public int Add() => 1 + 1; + + private class FailingBuildConfig : ManualConfig + { + public FailingBuildConfig() + => AddJob(Job.Dry.WithToolchain(new Toolchain("FailingBuild", new NoopGenerator(), new FailingBuilder(), new UnreachableExecutor()))); + } + + private sealed class NoopGenerator : IGenerator + { + public ValueTask GenerateProjectAsync(BuildPartition buildPartition, ILogger logger, string rootArtifactsFolderPath, CancellationToken cancellationToken) + => new(GenerateResult.Success(ArtifactsPaths.Empty, [])); + } + + private sealed class FailingBuilder : IBuilder + { + public ValueTask BuildAsync(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger, CancellationToken cancellationToken) + => new(BuildResult.Failure(generateResult, ErrorMessage)); + } + + private sealed class UnreachableExecutor : IExecutor + { + // A benchmark that failed to build is never executed. + public ValueTask ExecuteAsync(ExecuteParameters executeParameters, CancellationToken cancellationToken) + => throw new InvalidOperationException("The benchmark should never have been executed."); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/CollisionProbe.cs b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/CollisionProbe.cs new file mode 100644 index 0000000000..f40a600703 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/CollisionProbe.cs @@ -0,0 +1,36 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures +{ + /// + /// Two benchmark cases that BenchmarkDotNet identifies as one, because a benchmark is identified by the string + /// representation of its parameters and both values stringify the same way. The platform cannot tell the two apart + /// either, so the adapter is expected to report the collision instead of running them. + /// + [Config(typeof(FastConfig))] + public class CollisionProbe + { + public IEnumerable Values => [new Ambiguous(1), new Ambiguous(2)]; + + [ParamsSource(nameof(Values))] + public Ambiguous? Value { get; set; } + + [Benchmark] + public int Identity() => Value!.Number; + + public class Ambiguous(int number) + { + public int Number { get; } = number; + + public override string ToString() => "ambiguous"; + } + + private class FastConfig : ManualConfig + { + public FastConfig() => AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/README.md b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/README.md new file mode 100644 index 0000000000..1905d04e12 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures/README.md @@ -0,0 +1,9 @@ +# BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures + +Every benchmark in this Microsoft.Testing.Platform application is expected to be reported as failed, so a plain +`dotnet test` over it fails by design. It exists for the paths of `BenchmarkDotNet.TestAdapter` that only a broken +benchmark reaches: the uid collision report and the mapping of a build failure onto failed tests. + +`TestingPlatformAdapterTests` in `BenchmarkDotNet.IntegrationTests` drives it, one probe at a time, and asserts on +what the platform reports. The benchmarks that are expected to pass live in +`BenchmarkDotNet.IntegrationTests.TestingPlatform` instead. diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/BenchmarkDotNet.IntegrationTests.TestingPlatform.csproj b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/BenchmarkDotNet.IntegrationTests.TestingPlatform.csproj new file mode 100644 index 0000000000..cdbdbc7edc --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/BenchmarkDotNet.IntegrationTests.TestingPlatform.csproj @@ -0,0 +1,32 @@ + + + + net10.0 + Exe + BenchmarkDotNet.IntegrationTests.TestingPlatform + BenchmarkDotNet.IntegrationTests.TestingPlatform + BenchmarkDotNet.IntegrationTests.TestingPlatform + + + true + + + + + + + + + + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/DescribedProbe.cs b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/DescribedProbe.cs new file mode 100644 index 0000000000..72922bc2a7 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/DescribedProbe.cs @@ -0,0 +1,28 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace BenchmarkDotNet.IntegrationTests.TestingPlatform +{ + /// + /// Benchmarks whose display name comes from the description rather than the method name. + /// + [Config(typeof(FastConfig))] + public class DescribedProbe + { + [Params(1)] + public int Size { get; set; } + + [Benchmark(Description = "A described benchmark")] + public int Described() => Size; + + [Benchmark] + public int Undescribed() => Size; + + private class FastConfig : ManualConfig + { + public FastConfig() => AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/GenericProbe.cs b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/GenericProbe.cs new file mode 100644 index 0000000000..24ffa27b9c --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/GenericProbe.cs @@ -0,0 +1,25 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace BenchmarkDotNet.IntegrationTests.TestingPlatform +{ + /// + /// A generic benchmark, used to check how closed generic types are named and grouped by test runners. + /// + [Config(typeof(GenericProbeConfig))] + [GenericTypeArguments(typeof(int))] + [GenericTypeArguments(typeof(char))] + [GenericTypeArguments(typeof(System.Collections.Generic.List))] + public class GenericProbe where T : new() + { + [Benchmark] + public T Create() => new T(); + } + + internal class GenericProbeConfig : ManualConfig + { + public GenericProbeConfig() => AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)); + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/OutOfProcessProbe.cs b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/OutOfProcessProbe.cs new file mode 100644 index 0000000000..cba4a1748b --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/OutOfProcessProbe.cs @@ -0,0 +1,25 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; + +namespace BenchmarkDotNet.IntegrationTests.TestingPlatform +{ + /// + /// A benchmark that runs out of process, on the default toolchain. The other probes stay in process to keep the run + /// fast, which skips the generate/build/execute cycle entirely, so this one is what makes the adapter see a real + /// . + /// + [Config(typeof(OutOfProcessConfig))] + public class OutOfProcessProbe + { + [Benchmark] + public int Add() => 1 + 1; + + private class OutOfProcessConfig : ManualConfig + { + // A dry job on the default toolchain: one iteration, but a separate executable is still generated, built + // and run. + public OutOfProcessConfig() => AddJob(Job.Dry); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/SampleBenchmarks.cs b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/SampleBenchmarks.cs new file mode 100644 index 0000000000..27bce02ef5 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/SampleBenchmarks.cs @@ -0,0 +1,31 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace BenchmarkDotNet.IntegrationTests.TestingPlatform +{ + /// + /// Benchmarks used to exercise the Microsoft.Testing.Platform adapter end to end. They run in-process with a + /// single iteration so that a full run stays fast. + /// + [Config(typeof(FastConfig))] + public class SampleBenchmarks + { + [Params(1, 2)] + public int Size { get; set; } + + [Benchmark] + [BenchmarkCategory("Fast")] + public int Add() => Size + Size; + + [Benchmark] + [BenchmarkCategory("Slow")] + public int Multiply() => Size * Size; + + private class FastConfig : ManualConfig + { + public FastConfig() => AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/SeparatorProbe.cs b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/SeparatorProbe.cs new file mode 100644 index 0000000000..ab01cae901 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/SeparatorProbe.cs @@ -0,0 +1,26 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.InProcess.Emit; + +namespace BenchmarkDotNet.IntegrationTests.TestingPlatform +{ + /// + /// A benchmark whose parameter contains the character Microsoft.Testing.Platform uses to separate the levels of the + /// tree a --treenode-filter walks. It has to stay at the same level of that tree as every other benchmark. + /// + [Config(typeof(FastConfig))] + public class SeparatorProbe + { + [Params("a/b")] + public string Value { get; set; } = ""; + + [Benchmark] + public int Length() => Value.Length; + + private class FastConfig : ManualConfig + { + public FastConfig() => AddJob(Job.Dry.WithToolchain(InProcessEmitToolchain.Default)); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/global.json b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/global.json new file mode 100644 index 0000000000..3140116df3 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.TestingPlatform/global.json @@ -0,0 +1,5 @@ +{ + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 768cccaaa7..fbb0355c9c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -39,6 +39,15 @@ + + + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests/TestingPlatformAdapterTests.cs b/tests/BenchmarkDotNet.IntegrationTests/TestingPlatformAdapterTests.cs new file mode 100644 index 0000000000..2e8520bf77 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests/TestingPlatformAdapterTests.cs @@ -0,0 +1,258 @@ +#if NETCOREAPP +using BenchmarkDotNet.Detectors; +using System.Diagnostics; +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace BenchmarkDotNet.IntegrationTests +{ + /// + /// Drives the two Microsoft.Testing.Platform probe applications through their command line and asserts on what + /// BenchmarkDotNet.TestAdapter reports back. Everything here goes through a separate process on purpose: the + /// adapter's job is to keep a benchmark identifiable and addressable from the outside, and discovery and execution + /// are two different processes when a test runner drives it. + /// + public class TestingPlatformAdapterTests(ITestOutputHelper output) + { + private const string PassingProbes = "BenchmarkDotNet.IntegrationTests.TestingPlatform"; + private const string FailingProbes = "BenchmarkDotNet.IntegrationTests.TestingPlatform.Failures"; + + // Both probe projects are single targeted, see their .csproj files. + private const string ProbeTargetFramework = "net10.0"; + + // A run that has to build a benchmark pays for a restore and a build of the generated project. + private static readonly TimeSpan Timeout = TimeSpan.FromMinutes(10); + + [Fact] + public void EveryBenchmarkIsDiscoveredUnderItsOwnName() + { + string[] expected = + [ + // The description of a [Benchmark(Description = ...)] is what a user recognises it by, so it is used + // instead of the method name. Without one the method name is used, and the parameters are appended to + // both. + "DescribedProbe.'A described benchmark'(Size: 1)", + "DescribedProbe.Undescribed(Size: 1)", + + // A generic benchmark is named after the type arguments it was closed over. + "GenericProbe.Create", + "GenericProbe>.Create", + "GenericProbe.Create", + + "OutOfProcessProbe.Add", + "SampleBenchmarks.Add(Size: 1)", + "SampleBenchmarks.Add(Size: 2)", + "SampleBenchmarks.Multiply(Size: 1)", + "SampleBenchmarks.Multiply(Size: 2)", + "SeparatorProbe.Length(Value: \"a/b\")", + ]; + + var discovered = Discover(PassingProbes); + + Assert.Equal( + expected, + discovered.Select(test => test.DisplayName.Substring(PassingProbes.Length + 1)).OrderBy(name => name, StringComparer.Ordinal)); + + // The platform identifies a node by its uid, so two benchmarks sharing one cannot be told apart. + Assert.Equal(discovered.Count, discovered.Select(test => test.Uid).Distinct().Count()); + } + + [Fact] + public void TheUidOfABenchmarkIsTheSameInEveryProcess() + { + var first = Discover(PassingProbes).ToDictionary(test => test.Uid, test => test.DisplayName); + var second = Discover(PassingProbes).ToDictionary(test => test.Uid, test => test.DisplayName); + + Assert.Equal(first, second); + } + + [Fact] + public void ABenchmarkCanBeRunByTheUidItWasDiscoveredWith() + { + // This is the contract a test runner relies on: it discovers in one process and asks for a uid in another. + var uid = Discover(PassingProbes) + .Single(test => test.DisplayName.EndsWith("SampleBenchmarks.Add(Size: 2)", StringComparison.Ordinal)) + .Uid; + + var summary = RunAndSummarize(PassingProbes, "--filter-uid", uid); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Succeeded); + Assert.Equal(0, summary.Failed); + } + + [Fact] + public void ATreeNodeFilterMatchesTheCategoriesOfABenchmark() + { + // The categories are published as filterable properties, which is what makes this expression work. + var discovered = Discover(PassingProbes, "--treenode-filter", "/*/*/*/*[Category=Fast]"); + + Assert.Equal( + new[] { "SampleBenchmarks.Add(Size: 1)", "SampleBenchmarks.Add(Size: 2)" }, + discovered.Select(test => test.DisplayName.Substring(PassingProbes.Length + 1)).OrderBy(name => name, StringComparer.Ordinal)); + } + + [Fact] + public void ATreeNodeFilterMatchesTheClassAndTheMethodOfABenchmark() + { + var discovered = Discover(PassingProbes, "--treenode-filter", "/*/*/SampleBenchmarks/Multiply*"); + + Assert.Equal(2, discovered.Count); + Assert.All(discovered, test => Assert.Contains("SampleBenchmarks.Multiply", test.DisplayName, StringComparison.Ordinal)); + } + + [Fact] + public void ABenchmarkStaysAtTheSameLevelOfTheTreeWhenAParameterContainsTheSeparator() + { + // The platform splits the tree path on every '/' and never unescapes it, so a parameter containing one has + // to be encoded rather than escaped: otherwise the benchmark sits one level deeper and this filter, which + // matches every other benchmark, would miss it. + var discovered = Discover(PassingProbes, "--treenode-filter", "/*/*/SeparatorProbe/*"); + + Assert.Single(discovered); + Assert.Contains("SeparatorProbe.Length(Value: \"a/b\")", discovered[0].DisplayName, StringComparison.Ordinal); + } + + [Fact] + public void AnOutOfProcessBenchmarkIsBuiltAndRun() + { + // The only probe that is not pinned to an in-process toolchain, so the only one that makes the adapter see + // a real generate/build/execute cycle. + var summary = RunAndSummarize(PassingProbes, "--treenode-filter", "/*/*/OutOfProcessProbe/*"); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Succeeded); + Assert.Equal(0, summary.Failed); + } + + [Fact] + public void ABuildFailureIsReportedAsAFailedTest() + { + var (summary, standardOutput) = Run(FailingProbes, "--treenode-filter", "/*/*/BuildFailureProbe/*"); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + Assert.Contains("// Build Error: The build of this benchmark always fails, on purpose.", standardOutput, StringComparison.Ordinal); + } + + [Fact] + public void BenchmarksSharingAUidAreReportedAsOneFailedTest() + { + // Two benchmarks the platform cannot tell apart are published as a single node during discovery, and the + // collision is reported when they are asked to run. + Assert.Single(Discover(FailingProbes, "--treenode-filter", "/*/*/CollisionProbe/*")); + + var (summary, standardOutput) = Run(FailingProbes, "--treenode-filter", "/*/*/CollisionProbe/*"); + + Assert.Equal(1, summary.Total); + Assert.Equal(1, summary.Failed); + Assert.Contains("2 benchmarks are identified as", standardOutput, StringComparison.Ordinal); + } + + private IReadOnlyList Discover(string project, params string[] arguments) + { + var (exitCode, standardOutput) = Execute(project, ["--list-tests", "json", .. arguments]); + + Assert.Equal(0, exitCode); + + using var document = JsonDocument.Parse(standardOutput); + + return document.RootElement.GetProperty("tests") + .EnumerateArray() + .Select(test => new DiscoveredTest(test.GetProperty("uid").GetString()!, test.GetProperty("displayName").GetString()!)) + .ToArray(); + } + + private TestRunSummary RunAndSummarize(string project, params string[] arguments) => Run(project, arguments).Summary; + + private (TestRunSummary Summary, string StandardOutput) Run(string project, params string[] arguments) + { + var (_, standardOutput) = Execute(project, arguments); + + return (TestRunSummary.Parse(standardOutput), standardOutput); + } + + private (int ExitCode, string StandardOutput) Execute(string project, string[] arguments) + { + var application = GetProbeApplication(project); + var startInfo = new ProcessStartInfo(application) + { + WorkingDirectory = Path.GetDirectoryName(application), + RedirectStandardOutput = true, + RedirectStandardError = true, + UseShellExecute = false + }; + + // Progress reporting redraws the screen in place, which is noise once the output is redirected. + foreach (var argument in arguments.Concat(["--no-ansi", "--progress", "off"])) + startInfo.ArgumentList.Add(argument); + + var standardOutput = new StringBuilder(); + var standardError = new StringBuilder(); + + using var process = new Process { StartInfo = startInfo }; + process.OutputDataReceived += (_, e) => { if (e.Data != null) lock (standardOutput) standardOutput.AppendLine(e.Data); }; + process.ErrorDataReceived += (_, e) => { if (e.Data != null) lock (standardError) standardError.AppendLine(e.Data); }; + + process.Start(); + process.BeginOutputReadLine(); + process.BeginErrorReadLine(); + + if (!process.WaitForExit((int)Timeout.TotalMilliseconds)) + { + process.Kill(entireProcessTree: true); + throw new TimeoutException($"'{Path.GetFileName(application)} {string.Join(" ", arguments)}' did not finish within {Timeout}."); + } + + // Lets the redirected output be flushed before it is read. + process.WaitForExit(); + + output.WriteLine($"$ {application} {string.Join(" ", startInfo.ArgumentList)}"); + output.WriteLine(standardOutput.ToString()); + + if (standardError.Length > 0) + output.WriteLine($"stderr:{Environment.NewLine}{standardError}"); + + return (process.ExitCode, standardOutput.ToString()); + } + + private static string GetProbeApplication(string project) + { + // The tests run from /tests/BenchmarkDotNet.IntegrationTests/bin///, and + // the probes are built next to them, by the ProjectReferences of this project. + var binaries = new DirectoryInfo(AppContext.BaseDirectory); + var configuration = binaries.Parent!.Name; + var testsFolder = binaries.Parent!.Parent!.Parent!.Parent!.FullName; + + var fileName = OsDetector.IsWindows() ? $"{project}.exe" : project; + var path = Path.Combine(testsFolder, project, "bin", configuration, ProbeTargetFramework, fileName); + + if (!File.Exists(path)) + throw new FileNotFoundException($"The probe application was not built. Expected it at '{path}'.", path); + + return path; + } + + private sealed record DiscoveredTest(string Uid, string DisplayName); + + private sealed record TestRunSummary(int Total, int Failed, int Succeeded, int Skipped) + { + public static TestRunSummary Parse(string standardOutput) + { + // The platform ends a run with a block of " : " lines under "Test run summary:". + int Read(string name) + { + var match = Regex.Match(standardOutput, $@"^\s*{name}:\s*(?\d+)\s*$", RegexOptions.Multiline); + + return match.Success + ? int.Parse(match.Groups["count"].Value) + : throw new InvalidOperationException($"The test run did not report a '{name}' count.{Environment.NewLine}{standardOutput}"); + } + + return new TestRunSummary(Read("total"), Read("failed"), Read("succeeded"), Read("skipped")); + } + } + } +} +#endif