diff --git a/BenchmarkDotNet.slnx b/BenchmarkDotNet.slnx index 2d5a3cf0dd..9761f9f8cc 100644 --- a/BenchmarkDotNet.slnx +++ b/BenchmarkDotNet.slnx @@ -40,6 +40,7 @@ + diff --git a/build/cSpell.json b/build/cSpell.json index e90695fa7f..0e3a002ab1 100644 --- a/build/cSpell.json +++ b/build/cSpell.json @@ -9,6 +9,7 @@ "Autofac", "bitness", "corlib", + "crossgen", "Cygwin", "Diagnoser", "diagnosers", diff --git a/docs/articles/configs/toolchains.md b/docs/articles/configs/toolchains.md index 4a1ffde01f..c4d6c02ce2 100644 --- a/docs/articles/configs/toolchains.md +++ b/docs/articles/configs/toolchains.md @@ -104,37 +104,35 @@ The recommended way of running the benchmarks for multiple runtimes is to use th ## Custom .NET Core Runtime -We can run your benchmarks for custom `` if you want. All you need to do is to create custom toolchain by calling `CsProjCoreToolchain.From` method, which accepts `NetCoreAppSettings`. +You can build and run your benchmarks against a custom target framework moniker by creating a toolchain from a `NetCoreAppSettings` instance: ```cs public class MyConfig : ManualConfig { public MyConfig() { - Add(Job.Default.With( + AddJob(Job.Default.WithToolchain( CsProjCoreToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: "net8.0-windows", - runtimeFrameworkVersion: "8.0.101", - name: ".NET 8.0 Windows")))); + CoreRuntime.Core80, + new NetCoreAppSettings { TargetFrameworkMoniker = "net8.0-windows" }))); } } ``` -## Custom .NET Runtime - -It's possible to benchmark a private build of .NET Runtime. All you need to do is to define a job with the right version of `ClrRuntime`. +To pin a specific runtime pack version (the equivalent of setting `` in the project) pass it as an MSBuild argument. It is applied to the restore, build and publish steps and overrides any value coming from the host project: ```cs -BenchmarkSwitcher - .FromAssembly(typeof(Program).Assembly) - .Run(args, - DefaultConfig.Instance.AddJob( - Job.ShortRun.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0")))); +public class MyConfig : ManualConfig +{ + public MyConfig() + { + AddJob(Job.Default + .WithRuntime(CoreRuntime.Core80) + .WithMsBuildArguments("/p:RuntimeFrameworkVersion=8.0.101")); + } +} ``` -This sends the provided version as a `COMPLUS_Version` env var to the benchmarked process. - ## Custom dotnet cli path We internally use dotnet cli to build and run .NET Core executables. Sometimes it might be mandatory to use non-default dotnet cli path. An example scenario could be a comparison of RyuJit 32bit vs 64 bit. It required due this [limitation](https://github.com/dotnet/cli/issues/7532) of dotnet cli @@ -144,16 +142,20 @@ public class CustomPathsConfig : ManualConfig { public CustomPathsConfig() { - var dotnetCli32bit = NetCoreAppSettings - .NetCoreApp31 - .WithCustomDotNetCliPath(@"C:\Program Files (x86)\dotnet\dotnet.exe", "32 bit cli"); + var dotnetCli32bit = new NetCoreAppSettings + { + TargetFrameworkMoniker = "net8.0", + CliPath = new FileInfo(@"C:\Program Files (x86)\dotnet\dotnet.exe") + }; - var dotnetCli64bit = NetCoreAppSettings - .NetCoreApp31 - .WithCustomDotNetCliPath(@"C:\Program Files\dotnet\dotnet.exe", "64 bit cli"); + var dotnetCli64bit = new NetCoreAppSettings + { + TargetFrameworkMoniker = "net8.0", + CliPath = new FileInfo(@"C:\Program Files\dotnet\dotnet.exe") + }; - AddJob(Job.RyuJitX86.WithToolchain(CsProjCoreToolchain.From(dotnetCli32bit)).WithId("32 bit cli")); - AddJob(Job.RyuJitX64.WithToolchain(CsProjCoreToolchain.From(dotnetCli64bit)).WithId("64 bit cli")); + AddJob(Job.RyuJitX86.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, dotnetCli32bit)).WithId("32 bit cli")); + AddJob(Job.RyuJitX64.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, dotnetCli64bit)).WithId("64 bit cli")); } } ``` @@ -197,7 +199,7 @@ This is why you need to: - install [pre-requisites](https://docs.microsoft.com/en-us/dotnet/core/deploying/native-aot/#prerequisites) required by NativeAOT compiler - target .NET to be able to run NativeAOT benchmarks (example: `net7.0` in the .csproj file) - run the app as a .NET process (example: `dotnet run -c Release -f net7.0`). -- specify the NativeAOT runtime in an explicit way, either by using console line arguments `--runtimes nativeaot7.0` (the recommended approach), or by using`[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.With(NativeAotRuntime.Net70)`: +- specify the NativeAOT runtime in an explicit way, either by using console line arguments `--runtimes nativeaot7.0` (the recommended approach), or by using`[SimpleJob]` attribute or by using the fluent Job config API `Job.ShortRun.WithRuntime(NativeAotRuntime.Net70)`: ```cmd dotnet run -c Release -f net7.0 --runtimes nativeaot7.0 @@ -231,30 +233,27 @@ If you want to benchmark some particular version of NativeAOT (or from a differe ```cs var config = DefaultConfig.Instance .AddJob(Job.ShortRun - .WithToolchain(NativeAotToolchain.CreateBuilder() - .UseNuGet( - microsoftDotNetILCompilerVersion: "7.0.0-*", // the version goes here - nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json") // this address might change over time - .DisplayName("NativeAOT NuGet") - .TargetFrameworkMoniker("net7.0") - .ToToolchain())); + .WithToolchain(CsProjNativeAotToolchain.From( + NativeAotRuntime.Net70, // compiles the benchmarks as net7.0 + NativeAotSettings.Default.WithNuGet( + ilCompilerVersion: "7.0.0-*", // the version goes here + nuGetFeedUrl: "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet7/nuget/v3/index.json"))) // this address might change over time + .WithId("NativeAOT NuGet")); ``` -The builder allows to configure more settings: -- specify packages restore path by using `PackagesRestorePath($path)` -- rooting all application assemblies by using `RootAllApplicationAssemblies($bool)`. This is disabled by default. -- generating stack trace metadata by using `IlcGenerateStackTraceData($bool)`. This option is enabled by default. -- set optimization preference by using `IlcOptimizationPreference($value)`. The default is `Speed`, you can configure it to `Size` or nothing -- set instruction set for the target OS, architecture and hardware by using `IlcInstructionSet($value)`. By default BDN recognizes most of the instruction sets on your machine and enables them. +`NativeAotSettings` exposes the equivalent options as `init` properties (use a record initializer, e.g. `NativeAotSettings.Default with { OptimizationPreference = "Size" }`): +- `PackagesPath` — packages restore path. +- `GenerateStackTraceData` — generate stack trace metadata. Enabled by default. +- `OptimizationPreference` — `"Speed"` (default), `"Size"`, or `""` for none. +- `InstructionSet` — instruction set for the target OS, architecture and hardware. By default BDN recognizes most of the instruction sets on your machine and enables them. +- `RuntimeIdentifier` — the target runtime identifier (RID). BenchmarkDotNet supports [rd.xml](https://github.com/dotnet/runtime/blob/main/src/coreclr/nativeaot/docs/rd-xml-format.md) files. To get given file respected by BenchmarkDotNet you need to place it in the same folder as the project that defines benchmarks and name it `rd.xml` or in case of multiple files give them `.rd.xml` extension. The alternative to `rd.xml` files is annotating types with [DynamicallyAccessedMembers](https://devblogs.microsoft.com/dotnet/app-trimming-in-net-5/) attribute. -If given benchmark is not supported by NativeAOT, you need to apply `[AotFilter]` attribute for it. Example: +If a given benchmark is not supported by NativeAOT (e.g. it relies on runtime code generation), exclude it with a filter that checks the runtime. For example, to skip it only for NativeAOT: ```cs -[Benchmark] -[AotFilter("Not supported by design.")] -public object CreateInstanceNames() => System.Activator.CreateInstance(_assemblyName, _typeName); +config.AddFilter(new SimpleFilter(benchmark => benchmark.GetRuntime() is not NativeAotRuntime)); ``` ### Generated files @@ -284,7 +283,6 @@ cat .\BenchmarkDotNet.Autogenerated.csproj Exe net7.0 win-x64 - Job-KRLVKQ Job-KRLVKQ true @@ -337,11 +335,10 @@ or explicitly in the code: ```cs var config = DefaultConfig.Instance .AddJob(Job.ShortRun - .WithToolchain(NativeAotToolchain.CreateBuilder() - .UseLocalBuild(@"C:\Projects\runtime\artifacts\packages\Release\Shipping\") - .DisplayName("NativeAOT local build") - .TargetFrameworkMoniker("net7.0") - .ToToolchain())); + .WithToolchain(CsProjNativeAotToolchain.From( + NativeAotRuntime.Net70, // compiles the benchmarks as net7.0 + NativeAotSettings.Default.WithLocalBuild(new DirectoryInfo(@"C:\Projects\runtime\artifacts\packages\Release\Shipping\")))) + .WithId("NativeAOT local build")); ``` BenchmarkDotNet is going to follow [these instructrions](https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/nativeaot.md#building) to get it working for you. @@ -426,39 +423,3 @@ And that you have .NET 5 feed added to your `nuget.config` file: Now you should be able to run the Wasm benchmarks! [!include[IntroWasm](../samples/IntroWasm.md)] - -## MonoAotLLVM - -BenchmarkDotNet supports doing Mono AOT runs with both the Mono-Mini compiler and the Mono-LLVM compiler (which uses llvm on the back end). - -Using this tool chain requires the following flags: - -``` ---runtimes monoaotllvm ---aotcompilerpath ---customruntimepack -``` - -and optionally (defaults to mini) - -``` ---aotcompilermode -``` - -As of this writing, the mono aot compiler is not available as a seperate download or nuget package. Therefore, it is required to build the compiler in the [dotnet/runtime repository]. - -The compiler binary (mono-sgen) is built as part of the `mono` subset, so it can be built (along with the runtime pack) like so (in the root of [dotnet/runtime]). - -`./build.sh -subset mono+libs -c Release` - -The compiler binary should be generated here (modify for your platform): - -``` -/artifacts/obj/mono/OSX.x64.Release/mono/mini/mono-sgen -``` - -And the runtime pack should be generated here: - -``` -artifacts/bin/microsoft.netcore.app.runtime.osx-x64/Release/ -``` diff --git a/docs/articles/guides/console-args.md b/docs/articles/guides/console-args.md index eecfb42208..8a16380738 100644 --- a/docs/articles/guides/console-args.md +++ b/docs/articles/guides/console-args.md @@ -282,7 +282,6 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `--packages` The directory to restore packages to (optional). * `--coreRun` Path(s) to CoreRun (optional). * `--monoPath` Optional path to Mono which should be used for running benchmarks. -* `--clrVersion` Optional version of private CLR build used as the value of `COMPLUS_Version` env var. * `--ilCompilerVersion` Optional version of Microsoft.DotNet.ILCompiler which should be used to run with NativeAOT. Example: "7.0.0-preview.3.22123.2" * `--ilcPackages` Optional path to shipping packages produced by local dotnet/runtime build. Example: 'D:\projects\runtime\artifacts\packages\Release\Shipping\' * `--launchCount` How many times we should launch process with target benchmark. The default is 1. @@ -318,10 +317,8 @@ dotnet run -c Release -- --filter * --runtimes net6.0 net8.0 --statisticalTest 5 * `--jitTieringMode` (Default: Auto) Controls the behavior of the JIT stage when tiering is enabled. Auto/Force/Skip. * `--wasmEngine` (Default: v8) Specifies the executable (in PATH) or full path to a java script engine used to run the benchmarks, used by Wasm toolchain. * `--wasmArgs` (Default: --expose_wasm) Arguments for the javascript engine used by Wasm toolchain. -* `--customRuntimePack` Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently. -* `--AOTCompilerPath` Path to Mono AOT compiler, used for MonoAotLLVM. -* `--AOTCompilerMode` (Default: mini) Mono AOT compiler mode, either 'mini' or 'llvm' -* `--wasmRuntimeFlavor` (Default: Mono) Runtime flavor for WASM benchmarks: 'Mono' (default) uses the Mono runtime pack, 'CoreCLR' uses the CoreCLR runtime pack. +* `--customRuntimePack` Path to a custom runtime pack. Only used for wasm currently. +* `--AOTCompilerPath` Path to the crossgen2 compiler, used for ReadyToRun (R2R) benchmarks. * `--wasmProcessTimeout` (Default: 10) Maximum time in minutes to wait for a single WASM benchmark process to finish before force killing it. * `--noForcedGCs` Specifying would not forcefully induce any GCs. * `--evaluateOverhead` Specifies whether to run and evaluate overhead iterations. diff --git a/samples/BenchmarkDotNet.Samples/IntroCustomMono.cs b/samples/BenchmarkDotNet.Samples/IntroCustomMono.cs index c71dcf3180..a0baa39a86 100644 --- a/samples/BenchmarkDotNet.Samples/IntroCustomMono.cs +++ b/samples/BenchmarkDotNet.Samples/IntroCustomMono.cs @@ -1,8 +1,8 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Mono; namespace BenchmarkDotNet.Samples { @@ -28,10 +28,8 @@ private class Config : ManualConfig { public Config() { - AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( - "Mono x64", @"C:\Program Files\Mono\bin\mono.exe"))); - AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( - "Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe"))); + AddJob(Job.ShortRun.WithToolchain(RoslynMonoToolchain.From(new() { MonoPath = new(@"C:\Program Files\Mono\bin\mono.exe") }))); + AddJob(Job.ShortRun.WithToolchain(RoslynMonoToolchain.From(new() { MonoPath = new(@"C:\Program Files (x86)\Mono\bin\mono.exe") }))); } } @@ -49,19 +47,23 @@ public class IntroCustomMonoObjectStyleAot { private class Config : ManualConfig { - public void AddMono(string name, string mono_top_dir) + public void AddMono(string mono_top_dir) { var aot_compile_args = "--aot=llvm"; var mono_bcl = $@"{mono_top_dir}\lib\mono\4.5"; var mono_bin = $@"{mono_top_dir}\bin\mono.exe"; - AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( - name, mono_bin, aot_compile_args, mono_bcl))); + AddJob(Job.ShortRun.WithToolchain(RoslynMonoAotToolchain.From(new() + { + MonoPath = new(mono_bin), + MonoBclPath = new(mono_bcl), + AotArgs = aot_compile_args + }))); } public Config() { - AddMono("Mono x64", @"C:\Program Files\Mono"); - AddMono("Mono x86", @"C:\Program Files (x86)\Mono"); + AddMono(@"C:\Program Files\Mono"); + AddMono(@"C:\Program Files (x86)\Mono"); } } @@ -80,10 +82,14 @@ public static void Run() { BenchmarkRunner.Run(ManualConfig .CreateMinimumViable() - .AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( - "Mono x64", @"C:\Program Files\Mono\bin\mono.exe"))) - .AddJob(Job.ShortRun.WithRuntime(new MonoRuntime( - "Mono x86", @"C:\Program Files (x86)\Mono\bin\mono.exe")))); + .AddJob(Job.ShortRun.WithToolchain(RoslynMonoToolchain.From(new() + { + MonoPath = new(@"C:\Program Files\Mono\bin\mono.exe") + }))) + .AddJob(Job.ShortRun.WithToolchain(RoslynMonoToolchain.From(new() + { + MonoPath = new(@"C:\Program Files (x86)\Mono\bin\mono.exe") + })))); } [Benchmark] diff --git a/samples/BenchmarkDotNet.Samples/IntroDisassemblyAllJits.cs b/samples/BenchmarkDotNet.Samples/IntroDisassemblyAllJits.cs index ce9fb489a6..ae12d6acf7 100644 --- a/samples/BenchmarkDotNet.Samples/IntroDisassemblyAllJits.cs +++ b/samples/BenchmarkDotNet.Samples/IntroDisassemblyAllJits.cs @@ -13,16 +13,12 @@ public class MultipleJits : ManualConfig { public MultipleJits() { - AddJob(Job.ShortRun.WithPlatform(Platform.X86).WithRuntime(new MonoRuntime(name: "Mono x86", customPath: @"C:\Program Files (x86)\Mono\bin\mono.exe"))); - AddJob(Job.ShortRun.WithPlatform(Platform.X64).WithRuntime(new MonoRuntime(name: "Mono x64", customPath: @"C:\Program Files\Mono\bin\mono.exe"))); + AddJob(Job.ShortRun.WithJit(Jit.LegacyJit).WithPlatform(Platform.X86).WithRuntime(ClrRuntime.Net481)); + AddJob(Job.ShortRun.WithJit(Jit.LegacyJit).WithPlatform(Platform.X64).WithRuntime(ClrRuntime.Net481)); - AddJob(Job.ShortRun.WithJit(Jit.LegacyJit).WithPlatform(Platform.X86).WithRuntime(ClrRuntime.Net462)); - AddJob(Job.ShortRun.WithJit(Jit.LegacyJit).WithPlatform(Platform.X64).WithRuntime(ClrRuntime.Net462)); + AddJob(Job.ShortRun.WithJit(Jit.RyuJit).WithPlatform(Platform.X64).WithRuntime(ClrRuntime.Net481)); - AddJob(Job.ShortRun.WithJit(Jit.RyuJit).WithPlatform(Platform.X64).WithRuntime(ClrRuntime.Net462)); - - // RyuJit for .NET Core 5.0 - AddJob(Job.ShortRun.WithJit(Jit.RyuJit).WithPlatform(Platform.X64).WithRuntime(CoreRuntime.Core50)); + AddJob(Job.ShortRun.WithJit(Jit.RyuJit).WithPlatform(Platform.X64).WithRuntime(CoreRuntime.Core10_0)); AddDiagnoser(new DisassemblyDiagnoser(new DisassemblyDiagnoserConfig(maxDepth: 3, exportDiff: true))); } diff --git a/samples/BenchmarkDotNet.Samples/IntroWasm.cs b/samples/BenchmarkDotNet.Samples/IntroWasm.cs index 81ac17bebd..c334ff11a3 100644 --- a/samples/BenchmarkDotNet.Samples/IntroWasm.cs +++ b/samples/BenchmarkDotNet.Samples/IntroWasm.cs @@ -3,8 +3,7 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.Wasm; namespace BenchmarkDotNet.Samples { @@ -12,9 +11,9 @@ namespace BenchmarkDotNet.Samples public class IntroWasmCmdConfig { // Example: - // --runtimes wasmnet10.0 + // --runtimes monowasm10.0 // --cli /path/to/dotnet (optional) - // --wasmEngine v8 (optional) + // --wasmEngine node (optional) // --wasmArgs "--expose_wasm" (optional) // --wasmDataDir /path/to/data (optional) public static void Run(string[] args) => BenchmarkSwitcher.FromAssembly(typeof(IntroWasmCmdConfig).Assembly).Run(args); @@ -31,17 +30,10 @@ public class IntroWasmFluentConfig { public static void Run() { - // Optional: set this to use a custom `dotnet` (for example, a local dotnet/runtime build). - const string cliPath = ""; - - WasmRuntime runtime = new WasmRuntime(msBuildMoniker: "net10.0", RuntimeMoniker.WasmNet10_0, "Wasm .net10.0", false, "v8"); - NetCoreAppSettings netCoreAppSettings = new NetCoreAppSettings( - targetFrameworkMoniker: "net10.0", runtimeFrameworkVersion: "", name: "Wasm", - customDotNetCliPath: cliPath); - var toolChain = WasmToolchain.From(netCoreAppSettings); + var toolChain = CsProjMonoWasmToolchain.From(MonoWasmRuntime.Net10_0, WasmSettings.Default with { JavaScriptEngine = "node" }); BenchmarkRunner.Run(DefaultConfig.Instance - .AddJob(Job.ShortRun.WithRuntime(runtime).WithToolchain(toolChain))); + .AddJob(Job.ShortRun.WithToolchain(toolChain))); } [Benchmark] diff --git a/src/BenchmarkDotNet.Analyzers/AnalyzerReleases.Unshipped.md b/src/BenchmarkDotNet.Analyzers/AnalyzerReleases.Unshipped.md index d26a08b8a5..3c1d68169b 100644 --- a/src/BenchmarkDotNet.Analyzers/AnalyzerReleases.Unshipped.md +++ b/src/BenchmarkDotNet.Analyzers/AnalyzerReleases.Unshipped.md @@ -11,6 +11,7 @@ BDN1604 | Usage | Error | Properties annotated with [BenchmarkCancellatio BDN1605 | Usage | Info | Async benchmarks should have a [BenchmarkCancellation] property for cancellation support BDN1700 | Usage | Error | [GlobalSetup]/[GlobalCleanup]/[IterationSetup]/[IterationCleanup] method must not return an async enumerable BDN1701 | Usage | Warning | Benchmark/setup/cleanup return type is both awaitable and an async enumerable; the iterator is never enumerated +BDN1800 | Usage | Warning | Setting both Runtime and Toolchain on a job is order-dependent; they are coupled and the last assignment wins while the other is discarded ### Removed Rules diff --git a/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.Designer.cs b/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.Designer.cs index cbbf83fa08..71187fd2e5 100644 --- a/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.Designer.cs +++ b/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.Designer.cs @@ -1080,5 +1080,32 @@ internal static string General_AwaitableAsyncEnumerable_AmbiguousReturnType_Desc return ResourceManager.GetString("General_AwaitableAsyncEnumerable_AmbiguousReturnType_Description", resourceCulture); } } + + /// + /// Looks up a localized string similar to Setting both Runtime and Toolchain on a job is order-dependent. + /// + internal static string General_Job_RuntimeAndToolchainBothSet_Title { + get { + return ResourceManager.GetString("General_Job_RuntimeAndToolchainBothSet_Title", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to This job sets both a Runtime and a Toolchain. They are coupled, so whichever is assigned last takes effect and the other is silently discarded.... + /// + internal static string General_Job_RuntimeAndToolchainBothSet_MessageFormat { + get { + return ResourceManager.GetString("General_Job_RuntimeAndToolchainBothSet_MessageFormat", resourceCulture); + } + } + + /// + /// Looks up a localized string similar to The runtime and toolchain characteristics of a job are coupled.... + /// + internal static string General_Job_RuntimeAndToolchainBothSet_Description { + get { + return ResourceManager.GetString("General_Job_RuntimeAndToolchainBothSet_Description", resourceCulture); + } + } } } diff --git a/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.resx b/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.resx index 7194f15210..e057c10de8 100644 --- a/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.resx +++ b/src/BenchmarkDotNet.Analyzers/BenchmarkDotNetAnalyzerResources.resx @@ -451,4 +451,13 @@ Either add the [ArgumentsSource] or [Arguments] attribute(s) or remove the param When a benchmark or setup/cleanup return type satisfies both the awaitable pattern (a public GetAwaiter) and the async-enumerable pattern (a public GetAsyncEnumerator), BenchmarkDotNet treats it as awaitable — the iterator's body is never executed. Pick one shape so the intent is unambiguous: drop GetAwaiter if you want it consumed as an async enumerable, or drop GetAsyncEnumerator if you want it awaited. + + Setting both Runtime and Toolchain on a job is order-dependent + + + This job sets both a Runtime and a Toolchain. They are coupled, so whichever is assigned last takes effect and the other is silently discarded. Keep only the Toolchain (which determines the runtime); the fix removes the Runtime assignment + + + The runtime and toolchain characteristics of a job are coupled: setting the toolchain overwrites the runtime with the one the toolchain targets, and setting the runtime clears any explicitly set toolchain. When both are set on the same job, whichever assignment comes last wins and the other is silently discarded — so the outcome depends on ordering rather than on anything visible at the call site. Prefer setting only the Toolchain, which already determines the runtime; the default fix removes the Runtime assignment. If you instead intended the Runtime to win, remove the Toolchain. + \ No newline at end of file diff --git a/src/BenchmarkDotNet.Analyzers/DiagnosticIds.cs b/src/BenchmarkDotNet.Analyzers/DiagnosticIds.cs index 56eca0fb3b..497d939b0d 100644 --- a/src/BenchmarkDotNet.Analyzers/DiagnosticIds.cs +++ b/src/BenchmarkDotNet.Analyzers/DiagnosticIds.cs @@ -42,4 +42,5 @@ public static class DiagnosticIds public const string General_AsyncBenchmark_ShouldHaveCancellationToken = "BDN1605"; public const string Attributes_SetupCleanup_MustNotReturnAsyncEnumerable = "BDN1700"; public const string General_AwaitableAsyncEnumerable_AmbiguousReturnType = "BDN1701"; + public const string General_Job_RuntimeAndToolchainBothSet = "BDN1800"; } diff --git a/src/BenchmarkDotNet.Analyzers/General/RuntimeAndToolchainAnalyzer.cs b/src/BenchmarkDotNet.Analyzers/General/RuntimeAndToolchainAnalyzer.cs new file mode 100644 index 0000000000..84c3fe26d3 --- /dev/null +++ b/src/BenchmarkDotNet.Analyzers/General/RuntimeAndToolchainAnalyzer.cs @@ -0,0 +1,208 @@ +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using Microsoft.CodeAnalysis.Diagnostics; +using Microsoft.CodeAnalysis.Operations; +using System.Collections.Immutable; + +namespace BenchmarkDotNet.Analyzers.General; + +/// +/// Flags jobs that set both a runtime and a toolchain. The two are coupled (setting the toolchain overwrites the runtime, +/// and setting the runtime clears the toolchain), so specifying both is order-dependent and misleading: whichever is +/// assigned last wins and the other is silently discarded. The fix keeps the toolchain (the source of truth for the runtime). +/// Both the fluent form (.WithRuntime(...).WithToolchain(...), in any order) and explicit property assignments +/// (Infrastructure.Runtime = ... / Infrastructure.Toolchain = ..., including object initializers) are detected. +/// +[DiagnosticAnalyzer(LanguageNames.CSharp)] +public class RuntimeAndToolchainAnalyzer : DiagnosticAnalyzer +{ + // Passed through Diagnostic.Properties so the code fix knows how to remove the runtime assignment. + internal const string KindPropertyKey = "Kind"; + internal const string KindChain = "Chain"; + internal const string KindStatement = "Statement"; + internal const string KindInitializer = "Initializer"; + + internal static readonly DiagnosticDescriptor RuntimeAndToolchainBothSetRule = new( + DiagnosticIds.General_Job_RuntimeAndToolchainBothSet, + AnalyzerHelper.GetResourceString(nameof(BenchmarkDotNetAnalyzerResources.General_Job_RuntimeAndToolchainBothSet_Title)), + AnalyzerHelper.GetResourceString(nameof(BenchmarkDotNetAnalyzerResources.General_Job_RuntimeAndToolchainBothSet_MessageFormat)), + "Usage", + DiagnosticSeverity.Warning, + isEnabledByDefault: true, + description: AnalyzerHelper.GetResourceString(nameof(BenchmarkDotNetAnalyzerResources.General_Job_RuntimeAndToolchainBothSet_Description))); + + public override ImmutableArray SupportedDiagnostics => new DiagnosticDescriptor[] + { + RuntimeAndToolchainBothSetRule, + }.ToImmutableArray(); + + public override void Initialize(AnalysisContext analysisContext) + { + analysisContext.EnableConcurrentExecution(); + analysisContext.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None); + + analysisContext.RegisterCompilationStartAction(ctx => + { + // Only run if the coupled BenchmarkDotNet APIs are referenced. + var jobExtensionsSymbol = ctx.Compilation.GetTypeByMetadataName("BenchmarkDotNet.Jobs.JobExtensions"); + var infrastructureModeSymbol = ctx.Compilation.GetTypeByMetadataName("BenchmarkDotNet.Jobs.InfrastructureMode"); + if (jobExtensionsSymbol == null || infrastructureModeSymbol == null) + { + return; + } + + ctx.RegisterOperationBlockAction(blockContext => AnalyzeOperationBlock(blockContext, jobExtensionsSymbol, infrastructureModeSymbol)); + }); + } + + private sealed class GroupInfo + { + // Each runtime setter that should be reported when this group also has a toolchain setter. + public List<(Location Location, string Kind)> RuntimeSetters { get; } = []; + public bool HasToolchainSetter { get; set; } + } + + private static void AnalyzeOperationBlock(OperationBlockAnalysisContext context, INamedTypeSymbol jobExtensionsSymbol, INamedTypeSymbol infrastructureModeSymbol) + { + // Group runtime/toolchain setters that operate on the same job. The key identifies the shared job: + // * fluent chains -> the outermost invocation node of the chain (same node instance for every link), + // * object initializers -> the enclosing initializer node, + // * sequential property assignments -> the textual receiver (scoped to this operation block). + var groups = new Dictionary(); + + foreach (var operationBlock in context.OperationBlocks) + { + foreach (var operation in operationBlock.DescendantsAndSelf()) + { + switch (operation) + { + case IInvocationOperation invocation: + AnalyzeInvocation(invocation, jobExtensionsSymbol, groups); + break; + case ISimpleAssignmentOperation assignment: + AnalyzeAssignment(assignment, infrastructureModeSymbol, groups); + break; + } + } + } + + foreach (var group in groups.Values) + { + if (!group.HasToolchainSetter) + { + continue; + } + + foreach (var (location, kind) in group.RuntimeSetters) + { + context.ReportDiagnostic(Diagnostic.Create( + RuntimeAndToolchainBothSetRule, + location, + ImmutableDictionary.Create().Add(KindPropertyKey, kind))); + } + } + } + + private static void AnalyzeInvocation(IInvocationOperation invocation, INamedTypeSymbol jobExtensionsSymbol, Dictionary groups) + { + var method = invocation.TargetMethod; + if (!SymbolEqualityComparer.Default.Equals(method.ContainingType, jobExtensionsSymbol)) + { + return; + } + + bool isRuntime = method.Name == "WithRuntime"; + bool isToolchain = method.Name == "WithToolchain"; + if (!isRuntime && !isToolchain) + { + return; + } + + var group = GetOrAdd(groups, GetInvocationChainRoot(invocation.Syntax)); + if (isToolchain) + { + group.HasToolchainSetter = true; + } + else + { + group.RuntimeSetters.Add((GetInvocationNameLocation(invocation.Syntax), KindChain)); + } + } + + private static void AnalyzeAssignment(ISimpleAssignmentOperation assignment, INamedTypeSymbol infrastructureModeSymbol, Dictionary groups) + { + if (assignment.Target is not IPropertyReferenceOperation propertyReference + || !SymbolEqualityComparer.Default.Equals(propertyReference.Property.ContainingType, infrastructureModeSymbol)) + { + return; + } + + bool isRuntime = propertyReference.Property.Name == "Runtime"; + bool isToolchain = propertyReference.Property.Name == "Toolchain"; + if (!isRuntime && !isToolchain) + { + return; + } + + object key; + string kind; + if (propertyReference.Instance is IInstanceReferenceOperation) + { + // Object/collection initializer: `new InfrastructureMode { Runtime = ..., Toolchain = ... }` + // or `new Job { Infrastructure = { Runtime = ..., Toolchain = ... } }`. + var initializer = assignment.Syntax.FirstAncestorOrSelf(); + if (initializer == null) + { + return; + } + key = initializer; + kind = KindInitializer; + } + else + { + // Explicit receiver: `job.Infrastructure.Runtime = ...`. Group by the receiver text within this block. + key = "prop:" + (propertyReference.Instance?.Syntax.ToString() ?? string.Empty); + kind = KindStatement; + } + + var group = GetOrAdd(groups, key); + if (isToolchain) + { + group.HasToolchainSetter = true; + } + else + { + group.RuntimeSetters.Add((assignment.Syntax.GetLocation(), kind)); + } + } + + private static GroupInfo GetOrAdd(Dictionary groups, object key) + { + if (!groups.TryGetValue(key, out var group)) + { + group = new GroupInfo(); + groups[key] = group; + } + return group; + } + + // Walks up a fluent chain (`a.WithX(..).WithY(..)`) and returns the outermost invocation node, which is the same + // instance for every link, so all links in one chain share a grouping key. + private static SyntaxNode GetInvocationChainRoot(SyntaxNode invocation) + { + var current = invocation; + while (current.Parent is MemberAccessExpressionSyntax memberAccess + && memberAccess.Expression == current + && memberAccess.Parent is InvocationExpressionSyntax parentInvocation) + { + current = parentInvocation; + } + return current; + } + + // Squiggles just the `WithRuntime` name, falling back to the whole invocation if it isn't a member-access call. + private static Location GetInvocationNameLocation(SyntaxNode invocation) + => invocation is InvocationExpressionSyntax { Expression: MemberAccessExpressionSyntax memberAccess } + ? memberAccess.Name.GetLocation() + : invocation.GetLocation(); +} diff --git a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs index 2e7dbfb8cd..32307d419c 100644 --- a/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs +++ b/src/BenchmarkDotNet.Annotations/Jobs/RuntimeMoniker.cs @@ -1,245 +1,111 @@ namespace BenchmarkDotNet.Jobs { - public enum RuntimeMoniker + /// + /// Well-known runtime moniker strings for use with the job attributes (e.g. [SimpleJob(RuntimeMoniker.Net80)]) + /// and the --runtimes command line option. Any moniker string accepted by Runtime.Parse can also be used + /// directly, which allows targeting custom runtimes and arbitrary versions. + /// + public static class RuntimeMoniker { - /// - /// the same Runtime as the host Process (default setting) - /// - HostProcess = 0, - - /// - /// not recognized, possibly a new version of .NET Core - /// - NotRecognized, - - /// - /// Mono - /// - Mono, - - /// - /// .NET 4.6.1 - /// - Net461, - - /// - /// .NET 4.6.2 - /// - Net462, - - /// - /// .NET 4.7 - /// - Net47, - - /// - /// .NET 4.7.1 - /// - Net471, - - /// - /// .NET 4.7.2 - /// - Net472, - - /// - /// .NET 4.8 - /// - Net48, - - /// - /// .NET 4.8.1 - /// - Net481, - - /// - /// .NET Core 2.0 - /// - NetCoreApp20, - - /// - /// .NET Core 2.1 - /// - NetCoreApp21, - - /// - /// .NET Core 2.2 - /// - NetCoreApp22, - - /// - /// .NET Core 3.0 - /// - NetCoreApp30, - - /// - /// .NET Core 3.1 - /// - NetCoreApp31, - - /// - /// .NET 5.0 - /// - Net50, // it's after NetCoreApp50 in the enum definition because the value of enumeration is used for framework version comparison using > < operators - - /// - /// .NET 6.0 - /// - Net60, - - /// - /// .NET 7.0 - /// - Net70, - - /// - /// .NET 8.0 - /// - Net80, - - /// - /// .NET 9.0 - /// - Net90, - - /// - /// .NET 10.0 - /// - Net10_0, - - /// - /// .NET 11.0 - /// - Net11_0, - - /// - /// NativeAOT compiled as net7.0 - /// - NativeAot70, - - /// - /// NativeAOT compiled as net8.0 - /// - NativeAot80, - - /// - /// NativeAOT compiled as net9.0 - /// - NativeAot90, - - /// - /// NativeAOT compiled as net10.0 - /// - NativeAot10_0, - - /// - /// NativeAOT compiled as net11.0 - /// - NativeAot11_0, - - /// - /// WebAssembly with net8.0 - /// - WasmNet80, - - /// - /// WebAssembly with net9.0 - /// - WasmNet90, - - /// - /// WebAssembly with net10.0 - /// - WasmNet10_0, - - /// - /// WebAssembly with net11.0 - /// - WasmNet11_0, - - /// - /// Mono with the Ahead of Time LLVM Compiler backend - /// - MonoAOTLLVM, - - /// - /// Mono with the Ahead of Time LLVM Compiler backend and net6.0 - /// - MonoAOTLLVMNet60, - - /// - /// Mono with the Ahead of Time LLVM Compiler backend and net7.0 - /// - MonoAOTLLVMNet70, - - /// - /// Mono with the Ahead of Time LLVM Compiler backend and net8.0 - /// - MonoAOTLLVMNet80, - - /// - /// Mono with the Ahead of Time LLVM Compiler backend and net9.0 - /// - MonoAOTLLVMNet90, - - /// - /// Mono with the Ahead of Time LLVM Compiler backend and net10.0 - /// - MonoAOTLLVMNet10_0, - - /// - /// Mono with the Ahead of Time LLVM Compiler backend and net11.0 - /// - MonoAOTLLVMNet11_0, - - /// - /// .NET 6 using MonoVM (not CLR which is the default) - /// - Mono60, - - /// - /// .NET 7 using MonoVM (not CLR which is the default) - /// - Mono70, - - /// - /// .NET 8 using MonoVM (not CLR which is the default) - /// - Mono80, - - /// - /// .NET 9 using MonoVM (not CLR which is the default) - /// - Mono90, - - /// - /// .NET 10 using MonoVM (not CLR which is the default) - /// - Mono10_0, - - /// - /// .NET 11 using MonoVM (not CLR which is the default) - /// - Mono11_0, - - /// - /// .NET 8 CLR with composite ReadyToRun compilation - /// - R2R80, - - /// - /// .NET 9 CLR with composite ReadyToRun compilation - /// - R2R90, - - /// - /// .NET 10 CLR with composite ReadyToRun compilation - /// - R2R10_0, - - /// - /// .NET 11 CLR with composite ReadyToRun compilation - /// - R2R11_0, + /// Legacy Mono + public const string Mono = "mono"; + + /// Legacy Mono AOT + public const string MonoAot = "monoaot"; + + /// .NET Framework 4.6.1 + public const string Net461 = "net461"; + /// .NET Framework 4.6.2 + public const string Net462 = "net462"; + /// .NET Framework 4.7 + public const string Net47 = "net47"; + /// .NET Framework 4.7.1 + public const string Net471 = "net471"; + /// .NET Framework 4.7.2 + public const string Net472 = "net472"; + /// .NET Framework 4.8 + public const string Net48 = "net48"; + /// .NET Framework 4.8.1 + public const string Net481 = "net481"; + + /// .NET Core 2.0 + public const string NetCoreApp20 = "netcoreapp2.0"; + /// .NET Core 2.1 + public const string NetCoreApp21 = "netcoreapp2.1"; + /// .NET Core 2.2 + public const string NetCoreApp22 = "netcoreapp2.2"; + /// .NET Core 3.0 + public const string NetCoreApp30 = "netcoreapp3.0"; + /// .NET Core 3.1 + public const string NetCoreApp31 = "netcoreapp3.1"; + + /// .NET 5.0 + public const string Net50 = "net5.0"; + /// .NET 6.0 + public const string Net60 = "net6.0"; + /// .NET 7.0 + public const string Net70 = "net7.0"; + /// .NET 8.0 + public const string Net80 = "net8.0"; + /// .NET 9.0 + public const string Net90 = "net9.0"; + /// .NET 10.0 + public const string Net10_0 = "net10.0"; + /// .NET 11.0 + public const string Net11_0 = "net11.0"; + + /// NativeAOT compiled as net7.0 + public const string NativeAot70 = "nativeaot7.0"; + /// NativeAOT compiled as net8.0 + public const string NativeAot80 = "nativeaot8.0"; + /// NativeAOT compiled as net9.0 + public const string NativeAot90 = "nativeaot9.0"; + /// NativeAOT compiled as net10.0 + public const string NativeAot10_0 = "nativeaot10.0"; + /// NativeAOT compiled as net11.0 + public const string NativeAot11_0 = "nativeaot11.0"; + + /// .NET 6 using MonoVM (not CLR which is the default) + public const string Mono60 = "mono6.0"; + /// .NET 7 using MonoVM (not CLR which is the default) + public const string Mono70 = "mono7.0"; + /// .NET 8 using MonoVM (not CLR which is the default) + public const string Mono80 = "mono8.0"; + /// .NET 9 using MonoVM (not CLR which is the default) + public const string Mono90 = "mono9.0"; + /// .NET 10 using MonoVM (not CLR which is the default) + public const string Mono10_0 = "mono10.0"; + /// .NET 11 using MonoVM (not CLR which is the default) + public const string Mono11_0 = "mono11.0"; + + /// .NET 8 CLR with composite ReadyToRun compilation + public const string R2R80 = "r2r8.0"; + /// .NET 9 CLR with composite ReadyToRun compilation + public const string R2R90 = "r2r9.0"; + /// .NET 10 CLR with composite ReadyToRun compilation + public const string R2R10_0 = "r2r10.0"; + /// .NET 11 CLR with composite ReadyToRun compilation + public const string R2R11_0 = "r2r11.0"; + + /// Mono WebAssembly with net8.0 + public const string MonoWasm80 = "monowasm8.0"; + /// Mono WebAssembly with net9.0 + public const string MonoWasm90 = "monowasm9.0"; + /// Mono WebAssembly with net10.0 + public const string MonoWasm10_0 = "monowasm10.0"; + /// Mono WebAssembly with net11.0 + public const string MonoWasm11_0 = "monowasm11.0"; + + /// Mono WebAssembly AOT with net8.0 + public const string MonoWasmAot80 = "monowasmaot8.0"; + /// Mono WebAssembly AOT with net9.0 + public const string MonoWasmAot90 = "monowasmaot9.0"; + /// Mono WebAssembly AOT with net10.0 + public const string MonoWasmAot10_0 = "monowasmaot10.0"; + /// Mono WebAssembly AOT with net11.0 + public const string MonoWasmAot11_0 = "monowasmaot11.0"; + + // Still experimental + //public const string CoreWasm11_0 = "corewasm11.0"; } } diff --git a/src/BenchmarkDotNet.CodeFixers/RuntimeAndToolchainCodeFixProvider.cs b/src/BenchmarkDotNet.CodeFixers/RuntimeAndToolchainCodeFixProvider.cs new file mode 100644 index 0000000000..e984fa4c93 --- /dev/null +++ b/src/BenchmarkDotNet.CodeFixers/RuntimeAndToolchainCodeFixProvider.cs @@ -0,0 +1,161 @@ +using BenchmarkDotNet.Analyzers; +using Microsoft.CodeAnalysis; +using Microsoft.CodeAnalysis.CodeActions; +using Microsoft.CodeAnalysis.CodeFixes; +using Microsoft.CodeAnalysis.CSharp; +using Microsoft.CodeAnalysis.CSharp.Syntax; +using System.Collections.Immutable; +using System.Composition; + +namespace BenchmarkDotNet.CodeFixers; + +[ExportCodeFixProvider(LanguageNames.CSharp, Name = nameof(RuntimeAndToolchainCodeFixProvider)), Shared] +public class RuntimeAndToolchainCodeFixProvider : CodeFixProvider +{ + // Mirrors the constants on BenchmarkDotNet.Analyzers.General.RuntimeAndToolchainAnalyzer (that type isn't referenced here). + private const string KindPropertyKey = "Kind"; + private const string KindChain = "Chain"; + private const string KindStatement = "Statement"; + private const string KindInitializer = "Initializer"; + + public override ImmutableArray FixableDiagnosticIds => ImmutableArray.Create(DiagnosticIds.General_Job_RuntimeAndToolchainBothSet); + + public override FixAllProvider GetFixAllProvider() => WellKnownFixAllProviders.BatchFixer; + + public override async Task RegisterCodeFixesAsync(CodeFixContext context) + { + var root = await context.Document.GetSyntaxRootAsync(context.CancellationToken).ConfigureAwait(false); + if (root == null) + return; + + var diagnostic = context.Diagnostics.First(); + var kind = diagnostic.Properties.TryGetValue(KindPropertyKey, out var value) ? value : null; + + // Decided here rather than inside the fix: an offered action that changes nothing - or throws - is worse than + // no action, and FixAll would abort on it. An initializer element is always removable. + var node = root.FindNode(diagnostic.Location.SourceSpan, getInnermostNodeForTie: true); + + if (kind == KindChain) + { + var semanticModel = await context.Document.GetSemanticModelAsync(context.CancellationToken).ConfigureAwait(false); + var invocation = node.FirstAncestorOrSelf(); + if (invocation == null || !IsReducedExtensionCall(invocation, semanticModel, context.CancellationToken)) + return; + } + else if (kind == KindStatement && !IsRemovableStatement(node.FirstAncestorOrSelf())) + { + return; + } + + context.RegisterCodeFix( + CodeAction.Create( + title: "Remove the Runtime assignment (keep the Toolchain)", + createChangedDocument: c => RemoveRuntimeAssignmentAsync(context.Document, diagnostic.Location.SourceSpan, kind, c), + equivalenceKey: nameof(RuntimeAndToolchainCodeFixProvider)), + diagnostic); + } + + private static async Task RemoveRuntimeAssignmentAsync(Document document, Microsoft.CodeAnalysis.Text.TextSpan diagnosticSpan, string? kind, CancellationToken cancellationToken) + { + var root = await document.GetSyntaxRootAsync(cancellationToken).ConfigureAwait(false); + if (root == null) + return document; + + var node = root.FindNode(diagnosticSpan, getInnermostNodeForTie: true); + + SyntaxNode? newRoot = kind switch + { + KindChain => RemoveChainLink(root, node), + KindStatement => RemoveStatement(root, node), + KindInitializer => RemoveInitializerElement(root, node), + _ => null, + }; + + return newRoot == null ? document : document.WithSyntaxRoot(newRoot); + } + + /// + /// Whether the statement can be deleted outright. An embedded statement - a brace-less if body, a labeled + /// statement - cannot: it would leave the enclosing construct without one, and the syntax remover throws rather + /// than produce that. Removing it would mean synthesizing a replacement, so no fix is offered. + /// + private static bool IsRemovableStatement(ExpressionStatementSyntax? statement) + => statement?.Parent is BlockSyntax or SwitchSectionSyntax or GlobalStatementSyntax; + + /// + /// Whether the call is the instance form job.WithRuntime(r) rather than the static form + /// JobExtensions.WithRuntime(job, r), where the receiver is the type and dropping the call would + /// silently discard the job argument and produce code that does not compile. + /// + /// + /// A conditional-access chain (job?.WithRuntime(r)) is declined for the same reason: its receiver is a + /// MemberBindingExpressionSyntax, not part of the invocation, so the link cannot be dropped by replacing + /// the call with it. + /// + private static bool IsReducedExtensionCall(InvocationExpressionSyntax invocation, SemanticModel? semanticModel, CancellationToken cancellationToken) + => invocation.Expression is MemberAccessExpressionSyntax + && semanticModel?.GetSymbolInfo(invocation, cancellationToken).Symbol is IMethodSymbol { ReducedFrom: not null }; + + // `a.WithRuntime(r).WithToolchain(t)` / `a.WithToolchain(t).WithRuntime(r)` -> drops the `.WithRuntime(r)` link + // by replacing the WithRuntime invocation with its receiver expression. + private static SyntaxNode? RemoveChainLink(SyntaxNode root, SyntaxNode node) + { + var invocation = node.FirstAncestorOrSelf(); + if (invocation?.Expression is not MemberAccessExpressionSyntax memberAccess) + return null; + + return root.ReplaceNode(invocation, memberAccess.Expression.WithTriviaFrom(invocation)); + } + + // `job.Infrastructure.Runtime = r;` -> removes the whole statement. + private static SyntaxNode? RemoveStatement(SyntaxNode root, SyntaxNode node) + { + var statement = node.FirstAncestorOrSelf(); + if (!IsRemovableStatement(statement)) + return null; + + // Under top-level statements the assignment is wrapped in a GlobalStatement; removing the inner statement + // would leave the wrapper empty, which the syntax remover refuses. + return RemoveKeepingComments(root, statement!.Parent is GlobalStatementSyntax global ? global : statement); + } + + /// + /// Removes a node, keeping any comment written above it and any directive that would otherwise be orphaned. + /// + /// + /// A node's leading trivia ends with the indentation of its line, which would otherwise be prepended to whatever + /// follows on top of that line's own indentation - hence the trim, and the annotation to find the trimmed node + /// again. A comment on the SAME line is not kept: it annotates the node being removed. + /// + private static SyntaxNode RemoveKeepingComments(SyntaxNode root, SyntaxNode node) + { + var annotation = new SyntaxAnnotation(); + var trimmed = node + .WithLeadingTrivia(TrimTrailingWhitespace(node.GetLeadingTrivia())) + .WithAdditionalAnnotations(annotation); + + var withTrimmed = root.ReplaceNode(node, trimmed); + var target = withTrimmed.GetAnnotatedNodes(annotation).First(); + + // KeepLeadingTrivia only covers the leading trivia of the node's FIRST token. A directive in the leading + // trivia of an interior token is inside the removed span, so without KeepUnbalancedDirectives its partner + // outside the node is orphaned and the result does not compile. + return withTrimmed.RemoveNode(target, SyntaxRemoveOptions.KeepLeadingTrivia | SyntaxRemoveOptions.KeepUnbalancedDirectives)!; + } + + private static SyntaxTriviaList TrimTrailingWhitespace(SyntaxTriviaList trivia) + { + int end = trivia.Count; + while (end > 0 && trivia[end - 1].IsKind(SyntaxKind.WhitespaceTrivia)) + end--; + + return end == trivia.Count ? trivia : SyntaxFactory.TriviaList(trivia.Take(end)); + } + + // `new InfrastructureMode { Runtime = r, Toolchain = t }` -> removes the `Runtime = r` element (and its separator). + private static SyntaxNode? RemoveInitializerElement(SyntaxNode root, SyntaxNode node) + { + var assignment = node.FirstAncestorOrSelf(); + return assignment == null ? null : RemoveKeepingComments(root, assignment); + } +} diff --git a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs index 14aac90efa..ef83d75b3a 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotMemory/DotMemoryDiagnoser.cs @@ -1,7 +1,7 @@ using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Jobs; using JetBrains.Profiler.SelfApi; using System.Reflection; @@ -67,64 +67,13 @@ protected override string GetRunnerPath() return runnerPath; } - internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + internal override bool IsSupported(Runtime runtime) => runtime switch { - switch (runtimeMoniker) - { - case RuntimeMoniker.HostProcess: - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - case RuntimeMoniker.Net10_0: - case RuntimeMoniker.Net11_0: - case RuntimeMoniker.R2R80: - case RuntimeMoniker.R2R90: - case RuntimeMoniker.R2R10_0: - case RuntimeMoniker.R2R11_0: - return true; - case RuntimeMoniker.NotRecognized: - case RuntimeMoniker.Mono: - case RuntimeMoniker.NativeAot70: - case RuntimeMoniker.NativeAot80: - case RuntimeMoniker.NativeAot90: - case RuntimeMoniker.NativeAot10_0: - case RuntimeMoniker.NativeAot11_0: - case RuntimeMoniker.WasmNet80: - case RuntimeMoniker.WasmNet90: - case RuntimeMoniker.WasmNet10_0: - case RuntimeMoniker.WasmNet11_0: - case RuntimeMoniker.MonoAOTLLVM: - case RuntimeMoniker.MonoAOTLLVMNet60: - case RuntimeMoniker.MonoAOTLLVMNet70: - case RuntimeMoniker.MonoAOTLLVMNet80: - case RuntimeMoniker.MonoAOTLLVMNet90: - case RuntimeMoniker.MonoAOTLLVMNet10_0: - case RuntimeMoniker.MonoAOTLLVMNet11_0: - case RuntimeMoniker.Mono60: - case RuntimeMoniker.Mono70: - case RuntimeMoniker.Mono80: - case RuntimeMoniker.Mono90: - case RuntimeMoniker.Mono10_0: - case RuntimeMoniker.Mono11_0: - return false; - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - return OsDetector.IsWindows(); - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: - return OsDetector.IsWindows() || OsDetector.IsLinux(); - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); - } - } + ClrRuntime => true, + R2RRuntime => true, + CoreRuntime core when core.Version.Major < 3 => OsDetector.IsWindows(), + CoreRuntime core when core.Version.Major < 5 => OsDetector.IsWindows() || OsDetector.IsLinux(), + CoreRuntime => true, + _ => false, + }; } diff --git a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs index 2cf44411ae..762377102a 100644 --- a/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs +++ b/src/BenchmarkDotNet.Diagnostics.dotTrace/DotTraceDiagnoser.cs @@ -1,7 +1,7 @@ using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Diagnosers; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Jobs; using JetBrains.Profiler.SelfApi; using System.Reflection; @@ -70,64 +70,13 @@ protected override string GetRunnerPath() return runnerPath; } - internal override bool IsSupported(RuntimeMoniker runtimeMoniker) + internal override bool IsSupported(Runtime runtime) => runtime switch { - switch (runtimeMoniker) - { - case RuntimeMoniker.HostProcess: - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - case RuntimeMoniker.Net10_0: - case RuntimeMoniker.Net11_0: - case RuntimeMoniker.R2R80: - case RuntimeMoniker.R2R90: - case RuntimeMoniker.R2R10_0: - case RuntimeMoniker.R2R11_0: - return true; - case RuntimeMoniker.NotRecognized: - case RuntimeMoniker.Mono: - case RuntimeMoniker.NativeAot70: - case RuntimeMoniker.NativeAot80: - case RuntimeMoniker.NativeAot90: - case RuntimeMoniker.NativeAot10_0: - case RuntimeMoniker.NativeAot11_0: - case RuntimeMoniker.WasmNet80: - case RuntimeMoniker.WasmNet90: - case RuntimeMoniker.WasmNet10_0: - case RuntimeMoniker.WasmNet11_0: - case RuntimeMoniker.MonoAOTLLVM: - case RuntimeMoniker.MonoAOTLLVMNet60: - case RuntimeMoniker.MonoAOTLLVMNet70: - case RuntimeMoniker.MonoAOTLLVMNet80: - case RuntimeMoniker.MonoAOTLLVMNet90: - case RuntimeMoniker.MonoAOTLLVMNet10_0: - case RuntimeMoniker.MonoAOTLLVMNet11_0: - case RuntimeMoniker.Mono60: - case RuntimeMoniker.Mono70: - case RuntimeMoniker.Mono80: - case RuntimeMoniker.Mono90: - case RuntimeMoniker.Mono10_0: - case RuntimeMoniker.Mono11_0: - return false; - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - return OsDetector.IsWindows(); - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: - return OsDetector.IsWindows() || OsDetector.IsLinux(); - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, $"Runtime moniker {runtimeMoniker} is not supported"); - } - } + ClrRuntime => true, + R2RRuntime => true, + CoreRuntime core when core.Version.Major < 3 => OsDetector.IsWindows(), + CoreRuntime core when core.Version.Major < 5 => OsDetector.IsWindows() || OsDetector.IsLinux(), + CoreRuntime => true, + _ => false, + }; } diff --git a/src/BenchmarkDotNet/Attributes/Filters/AotFilterAttribute.cs b/src/BenchmarkDotNet/Attributes/Filters/AotFilterAttribute.cs deleted file mode 100644 index 1bf350dd96..0000000000 --- a/src/BenchmarkDotNet/Attributes/Filters/AotFilterAttribute.cs +++ /dev/null @@ -1,12 +0,0 @@ -using BenchmarkDotNet.Filters; - -namespace BenchmarkDotNet.Attributes.Filters -{ - public class AotFilterAttribute : FilterConfigBaseAttribute - { - public AotFilterAttribute(string? reason = null) - : base(new SimpleFilter(benchmark => !benchmark.GetRuntime().IsAOT)) - { - } - } -} diff --git a/src/BenchmarkDotNet/Attributes/Jobs/DryJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/DryJobAttribute.cs index a8a14fa830..d8522d1348 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/DryJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/DryJobAttribute.cs @@ -14,7 +14,7 @@ public DryJobAttribute() : base(Job.Dry) /// defines a new Dry Job that targets specified Framework /// /// Target Framework to test. - public DryJobAttribute(RuntimeMoniker runtimeMoniker) + public DryJobAttribute(string runtimeMoniker) : base(GetJob(Job.Dry, runtimeMoniker, null, null)) { } @@ -25,7 +25,7 @@ public DryJobAttribute(RuntimeMoniker runtimeMoniker) /// Target Framework to test. /// Jit to test. /// Platform to test. - public DryJobAttribute(RuntimeMoniker runtimeMoniker, Jit jit, Platform platform) + public DryJobAttribute(string runtimeMoniker, Jit jit, Platform platform) : base(GetJob(Job.Dry, runtimeMoniker, jit, platform)) { } diff --git a/src/BenchmarkDotNet/Attributes/Jobs/InProcessAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/InProcessAttribute.cs index 02eaa1f6d7..1c7e4f64dc 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/InProcessAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/InProcessAttribute.cs @@ -34,8 +34,8 @@ internal static Job GetJob(Job baseJob, InProcessToolchainType toolchainType, bo IToolchain toolchain = toolchainType switch { - InProcessToolchainType.Emit => new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = executeOnSeparateThread }), - InProcessToolchainType.NoEmit => new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = executeOnSeparateThread }), + InProcessToolchainType.Emit => InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = executeOnSeparateThread }), + InProcessToolchainType.NoEmit => InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = executeOnSeparateThread }), _ => throw new ArgumentOutOfRangeException(nameof(toolchainType)) }; diff --git a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs index 0ece453a6e..1eb3867198 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/JobConfigbaseAttribute.cs @@ -1,6 +1,5 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using JetBrains.Annotations; @@ -17,10 +16,10 @@ public class JobConfigBaseAttribute : Attribute, IConfigSource public IConfig Config { get; } - protected static Job GetJob(Job sourceJob, RuntimeMoniker runtimeMoniker, Jit? jit, Platform? platform) + protected static Job GetJob(Job sourceJob, string runtimeMoniker, Jit? jit, Platform? platform) { - var runtime = runtimeMoniker.GetRuntime(); - var baseJob = sourceJob.WithRuntime(runtime).WithId($"{sourceJob.Id}-{runtime.Name}"); + var runtime = Runtime.Parse(runtimeMoniker); + var baseJob = sourceJob.WithRuntime(runtime).WithId($"{sourceJob.Id}-{runtime}"); var id = baseJob.Id; if (jit.HasValue) diff --git a/src/BenchmarkDotNet/Attributes/Jobs/LongRunJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/LongRunJobAttribute.cs index d0c49c8911..4b89cdd227 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/LongRunJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/LongRunJobAttribute.cs @@ -14,7 +14,7 @@ public LongRunJobAttribute() : base(Job.LongRun) /// defines a new LongRun Job that targets specified Framework /// /// Target Framework to test. - public LongRunJobAttribute(RuntimeMoniker runtimeMoniker) + public LongRunJobAttribute(string runtimeMoniker) : base(GetJob(Job.LongRun, runtimeMoniker, null, null)) { } @@ -25,7 +25,7 @@ public LongRunJobAttribute(RuntimeMoniker runtimeMoniker) /// Target Framework to test. /// Jit to test. /// Platform to test. - public LongRunJobAttribute(RuntimeMoniker runtimeMoniker, Jit jit, Platform platform) + public LongRunJobAttribute(string runtimeMoniker, Jit jit, Platform platform) : base(GetJob(Job.LongRun, runtimeMoniker, jit, platform)) { } diff --git a/src/BenchmarkDotNet/Attributes/Jobs/MediumRunJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/MediumRunJobAttribute.cs index b91191b53b..200afd0659 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/MediumRunJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/MediumRunJobAttribute.cs @@ -14,7 +14,7 @@ public MediumRunJobAttribute() : base(Job.MediumRun) /// defines a new MediumRun Job that targets specified Framework /// /// Target Framework to test. - public MediumRunJobAttribute(RuntimeMoniker runtimeMoniker) + public MediumRunJobAttribute(string runtimeMoniker) : base(GetJob(Job.MediumRun, runtimeMoniker, null, null)) { } @@ -25,7 +25,7 @@ public MediumRunJobAttribute(RuntimeMoniker runtimeMoniker) /// Target Framework to test. /// Jit to test. /// Platform to test. - public MediumRunJobAttribute(RuntimeMoniker runtimeMoniker, Jit jit, Platform platform) + public MediumRunJobAttribute(string runtimeMoniker, Jit jit, Platform platform) : base(GetJob(Job.MediumRun, runtimeMoniker, jit, platform)) { } diff --git a/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs index e27863f8b5..eece8d46fb 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/MonoJobAttribute.cs @@ -1,6 +1,6 @@ using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains.Mono; namespace BenchmarkDotNet.Attributes { @@ -11,12 +11,12 @@ public MonoJobAttribute(bool baseline = false) : base(Job.Default.WithRuntime(Mo { } - public MonoJobAttribute(RuntimeMoniker runtimeMoniker, bool baseline = false) : base(Job.Default.WithRuntime(runtimeMoniker.GetRuntime()).WithBaseline(baseline)) + public MonoJobAttribute(string runtimeMoniker, bool baseline = false) : base(Job.Default.WithRuntime(Runtime.Parse(runtimeMoniker)).WithBaseline(baseline)) { } public MonoJobAttribute(string name, string path, bool baseline = false) - : base(new Job(name, new EnvironmentMode(new MonoRuntime(name, path)).Freeze()).WithBaseline(baseline).Freeze()) + : base(new Job(name).WithToolchain(RoslynMonoToolchain.From(new() { MonoPath = new(path) })).WithBaseline(baseline).Freeze()) { } } diff --git a/src/BenchmarkDotNet/Attributes/Jobs/ShortRunJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/ShortRunJobAttribute.cs index a3fbd88993..f0c08f9cf6 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/ShortRunJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/ShortRunJobAttribute.cs @@ -14,7 +14,7 @@ public ShortRunJobAttribute() : base(Job.ShortRun) /// defines a new ShortRun Job that targets specified Framework /// /// Target Framework to test. - public ShortRunJobAttribute(RuntimeMoniker runtimeMoniker) + public ShortRunJobAttribute(string runtimeMoniker) : base(GetJob(Job.ShortRun, runtimeMoniker, null, null)) { } @@ -25,7 +25,7 @@ public ShortRunJobAttribute(RuntimeMoniker runtimeMoniker) /// Target Framework to test. /// Jit to test. /// Platform to test. - public ShortRunJobAttribute(RuntimeMoniker runtimeMoniker, Jit jit, Platform platform) + public ShortRunJobAttribute(string runtimeMoniker, Jit jit, Platform platform) : base(GetJob(Job.ShortRun, runtimeMoniker, jit, platform)) { } diff --git a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs index e86083e2e3..f30cd717f2 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/SimpleJobAttribute.cs @@ -34,7 +34,7 @@ public SimpleJobAttribute( [PublicAPI] public SimpleJobAttribute( - RuntimeMoniker runtimeMoniker, + string runtimeMoniker, int launchCount = DefaultValue, int warmupCount = DefaultValue, int iterationCount = DefaultValue, @@ -46,7 +46,7 @@ public SimpleJobAttribute( [PublicAPI] public SimpleJobAttribute( RunStrategy runStrategy, - RuntimeMoniker runtimeMoniker, + string runtimeMoniker, int launchCount = DefaultValue, int warmupCount = DefaultValue, int iterationCount = DefaultValue, @@ -56,7 +56,7 @@ public SimpleJobAttribute( ) : base(CreateJob(id, launchCount, warmupCount, iterationCount, invocationCount, runStrategy, baseline, runtimeMoniker)) { } private static Job CreateJob(string id, int launchCount, int warmupCount, int iterationCount, int invocationCount, RunStrategy? runStrategy, - bool baseline, RuntimeMoniker runtimeMoniker = RuntimeMoniker.HostProcess) + bool baseline, string? runtimeMoniker = null) { var job = new Job(id); int manualValuesCount = 0; @@ -100,14 +100,14 @@ private static Job CreateJob(string id, int launchCount, int warmupCount, int it if (baseline) job.Meta.Baseline = true; - if (runtimeMoniker != RuntimeMoniker.HostProcess) + if (runtimeMoniker.IsNotBlank()) { - job.Environment.Runtime = runtimeMoniker.GetRuntime(); + job.Infrastructure.Runtime = Runtime.Parse(runtimeMoniker); manualValuesCount++; } - if (id == null && manualValuesCount == 1 && runtimeMoniker != RuntimeMoniker.HostProcess) - job = job.WithId(runtimeMoniker.GetRuntime().Name); + if (id == null && manualValuesCount == 1 && runtimeMoniker.IsNotBlank()) + job = job.WithId(Runtime.Parse(runtimeMoniker).ToString()); return job.Freeze(); } diff --git a/src/BenchmarkDotNet/Attributes/Jobs/VeryLongRunJobAttribute.cs b/src/BenchmarkDotNet/Attributes/Jobs/VeryLongRunJobAttribute.cs index 178c85b250..22368e7047 100644 --- a/src/BenchmarkDotNet/Attributes/Jobs/VeryLongRunJobAttribute.cs +++ b/src/BenchmarkDotNet/Attributes/Jobs/VeryLongRunJobAttribute.cs @@ -16,7 +16,7 @@ public VeryLongRunJobAttribute() : base(Job.VeryLongRun) /// defines a new VeryLongRun Job that targets specified Framework /// /// Target Framework to test. - public VeryLongRunJobAttribute(RuntimeMoniker runtimeMoniker) + public VeryLongRunJobAttribute(string runtimeMoniker) : base(GetJob(Job.VeryLongRun, runtimeMoniker, null, null)) { } @@ -27,7 +27,7 @@ public VeryLongRunJobAttribute(RuntimeMoniker runtimeMoniker) /// Target Framework to test. /// Jit to test. /// Platform to test. - public VeryLongRunJobAttribute(RuntimeMoniker runtimeMoniker, Jit jit, Platform platform) + public VeryLongRunJobAttribute(string runtimeMoniker, Jit jit, Platform platform) : base(GetJob(Job.VeryLongRun, runtimeMoniker, jit, platform)) { } diff --git a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs index be3a446793..db5e741c49 100644 --- a/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs +++ b/src/BenchmarkDotNet/Columns/DefaultColumnProvider.cs @@ -1,6 +1,7 @@ using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Mathematics; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Toolchains; using JetBrains.Annotations; using Perfolizer.Mathematics.Common; @@ -10,11 +11,12 @@ public static class DefaultColumnProviders { [PublicAPI] public static readonly IColumnProvider Descriptor = new DescriptorColumnProvider(); [PublicAPI] public static readonly IColumnProvider Job = new JobColumnProvider(); + [PublicAPI] public static readonly IColumnProvider Settings = new SettingsColumnProvider(); [PublicAPI] public static readonly IColumnProvider Statistics = new StatisticsColumnProvider(); [PublicAPI] public static readonly IColumnProvider Params = new ParamsColumnProvider(); [PublicAPI] public static readonly IColumnProvider Metrics = new MetricsColumnProvider(); - public static readonly IColumnProvider[] Instance = [Descriptor, Job, Statistics, Params, Metrics]; + public static readonly IColumnProvider[] Instance = [Descriptor, Job, Settings, Statistics, Params, Metrics]; private class DescriptorColumnProvider : IColumnProvider { @@ -33,6 +35,27 @@ private class JobColumnProvider : IColumnProvider public IEnumerable GetColumns(Summary summary) => JobCharacteristicColumn.AllColumns; } + private class SettingsColumnProvider : IColumnProvider + { + // Ask each toolchain's settings which values to surface, then create one column per distinct key. + // Settings types that expose the same key share the column, so headers are never duplicated. + public IEnumerable GetColumns(Summary summary) + { + var keys = new HashSet(); + foreach (var benchmarkCase in summary.BenchmarksCases) + { + if ((benchmarkCase.GetToolchain() as IHasSettings)?.Settings is not { } settings) + continue; + var filled = new Dictionary(); + settings.FillSettings(filled); + keys.UnionWith(filled.Keys); + } + return keys + .OrderBy(key => key, StringComparer.Ordinal) + .Select(key => (IColumn)new SettingsColumn(key)); + } + } + private class StatisticsColumnProvider : IColumnProvider { public IEnumerable GetColumns(Summary summary) diff --git a/src/BenchmarkDotNet/Columns/SettingsColumn.cs b/src/BenchmarkDotNet/Columns/SettingsColumn.cs new file mode 100644 index 0000000000..ce38f08636 --- /dev/null +++ b/src/BenchmarkDotNet/Columns/SettingsColumn.cs @@ -0,0 +1,76 @@ +using System.Collections.Generic; +using System.Linq; +using BenchmarkDotNet.Parameters; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Columns; + +/// +/// Displays a single toolchain-settings value in the summary table. Settings types that expose the same key +/// (e.g. CliPath from the shared base) share one column. +/// +/// The column is only shown when the value differs between the benchmarks that actually have the setting. +/// A benchmark whose toolchain lacks the setting renders as NA and does not affect that decision, so a +/// setting is not reported just because another toolchain lacks it. A null value renders as ?. +/// +/// +public class SettingsColumn : IColumn +{ + // Distinct marker for "this benchmark's settings don't expose the key"; rendered as NA and excluded from the + // variance check. Kept separate from a null value (rendered as "?"), which means the setting exists but is unset. + private static readonly object NotApplicable = new(); + private const string NotApplicableText = "NA"; + + private readonly string key; + + public SettingsColumn(string key) + { + this.key = key; + Id = $"Settings.{key}"; + ColumnName = key; + Legend = $"Value of the '{key}' toolchain setting"; + } + + public string Id { get; } + public string ColumnName { get; } + public string Legend { get; } + public bool AlwaysShow => false; + public ColumnCategory Category => ColumnCategory.Job; + public int PriorityInCategory => 1; // after the job characteristic columns, which use 0 + public bool IsNumeric => false; + public UnitType UnitType => UnitType.Dimensionless; + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase); + + public string GetValue(Summary summary, BenchmarkCase benchmarkCase) + { + object? value = GetRawValue(benchmarkCase); + return ReferenceEquals(value, NotApplicable) + ? NotApplicableText + : value?.ToString() ?? ParameterInstance.NullParameterTextRepresentation; + } + + public bool IsDefault(Summary summary, BenchmarkCase benchmarkCase) => ReferenceEquals(GetRawValue(benchmarkCase), NotApplicable); + + // Only show when the value varies among the benchmarks that actually have this setting; NA benchmarks don't count. + // Values are compared by object equality, so implementations must emit value-comparable representations. + public bool IsAvailable(Summary summary) + => summary.BenchmarksCases + .Select(GetRawValue) + .Where(value => !ReferenceEquals(value, NotApplicable)) + .Distinct() + .Take(2) + .Count() > 1; + + private object? GetRawValue(BenchmarkCase benchmarkCase) + { + if ((benchmarkCase.GetToolchain() as IHasSettings)?.Settings is not { } settings) + return NotApplicable; + + var filled = new Dictionary(); + settings.FillSettings(filled); + return filled.TryGetValue(key, out object? value) ? value : NotApplicable; + } +} diff --git a/src/BenchmarkDotNet/Configs/DefaultConfig.cs b/src/BenchmarkDotNet/Configs/DefaultConfig.cs index df47aeb52f..f84712834d 100644 --- a/src/BenchmarkDotNet/Configs/DefaultConfig.cs +++ b/src/BenchmarkDotNet/Configs/DefaultConfig.cs @@ -80,7 +80,6 @@ public IEnumerable GetValidators() yield return ParamsAllValuesValidator.FailOnError; yield return ParamsValidator.FailOnError; yield return BenchmarkCancellationValidator.FailOnError; - yield return RuntimeValidator.DontFailOnError; } public IOrderer? Orderer => null; @@ -103,9 +102,9 @@ public string ArtifactsPath get { string root; - if (OsDetector.IsAndroid() || OsDetector.IsIOS()) + if (OsDetector.IsMobile()) { - // On mobile platforms (Android, iOS, and Mac Catalyst), use a writable location + // On mobile platforms (Android, iOS, tvOS), use a writable location // because the app bundle and current directory are read-only due to sandboxing root = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); } diff --git a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs index 18c1b65fd6..e7208faeef 100644 --- a/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs +++ b/src/BenchmarkDotNet/Configs/ImmutableConfigBuilder.cs @@ -242,12 +242,53 @@ private static IReadOnlyList GetRunnableJobs(IEnumerable jobs) var copy = result[i].UnfreezeCopy(); copy.Apply(mutatorJob); + ApplyCouplingFromMutator(copy, mutatorJob); result[i] = copy.Freeze(); } } - return result; + // Deduplicated again: the pass at the top of this method runs before the mutators are merged, so it + // cannot see jobs that only become identical here. + return result + .Select(ReconcileRuntimeAndToolchain) + .Distinct(JobComparer.Default) + .ToArray(); + } + + /// + /// Re-applies the Runtime/Toolchain coupling in the direction the mutator asked for. + /// + /// + /// Apply writes characteristics directly, past the setters that keep + /// the pair in sync, so a merged job can end up with the two disagreeing. The mutator is the later and more + /// specific instruction, so whichever of them it set wins - as it would through the setters. + /// + private static void ApplyCouplingFromMutator(Job job, Job mutatorJob) + { + if (mutatorJob.Infrastructure.HasValue(InfrastructureMode.ToolchainCharacteristic)) + InfrastructureMode.RuntimeCharacteristic[job.Infrastructure] = job.Infrastructure.Toolchain?.Runtime; + else if (mutatorJob.Infrastructure.HasValue(InfrastructureMode.RuntimeCharacteristic)) + InfrastructureMode.ToolchainCharacteristic[job.Infrastructure] = null; + } + + /// + /// Keeps a job's Runtime in sync with an explicitly set Toolchain, where nothing says which was meant to win. + /// + /// + /// The public characteristic indexers write past the setters too. With no mutator to say which assignment came + /// later, the toolchain wins: it is what builds and runs the benchmark, so a stale runtime would report one + /// nothing ran on. + /// + private static Job ReconcileRuntimeAndToolchain(Job job) + { + var toolchain = job.Infrastructure.Toolchain; + if (toolchain == null || Equals(job.Infrastructure.Runtime, toolchain.Runtime)) + return job; + + var copy = job.UnfreezeCopy(); + InfrastructureMode.RuntimeCharacteristic[copy.Infrastructure] = toolchain.Runtime; + return copy.Freeze(); } private class TypeComparer : IEqualityComparer diff --git a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs index cc06b11216..9b98c15bb7 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/CommandLineOptions.cs @@ -4,7 +4,6 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; using CommandLine; using CommandLine.Text; using JetBrains.Annotations; @@ -23,7 +22,10 @@ public class CommandLineOptions [Option('j', "job", Required = false, Default = "Default", HelpText = "Dry/Short/Medium/Long or Default")] public string BaseJob { get; set; } = ""; - [Option('r', "runtimes", Required = false, HelpText = "Full target framework moniker for .NET Core and .NET. For Mono just 'Mono'. For NativeAOT please append target runtime version (example: 'nativeaot7.0'). First one will be marked as baseline!")] + [Option('r', "runtimes", Required = false, HelpText = + "Full target framework moniker for (Core)CLR (e.g. 'net481', 'net8.0', 'net10.0-windows'). For legacy Mono just 'Mono' or 'MonoAot'." + + " Other supported runtimes: 'monoX.0' (.NET on MonoVM), 'nativeaotX.0', 'r2rX.0', 'monowasmX.0', 'monowasmaotX.0', 'corewasmX.0'." + + " First one will be marked as baseline!")] public IEnumerable Runtimes { get; set; } = []; [Option('e', "exporters", Required = false, HelpText = "GitHub/StackOverflow/RPlot/CSV/JSON/HTML/XML/CSVMeasurements/Markdown/Atlassian/Plain/BriefJSON/FullJSON/Asciidoc/BriefXML/FullXML/OpenMetrics. A custom IExporter can also be selected by its assembly-qualified type name, e.g. \"My.Namespace.MyExporter, MyAssembly\".")] @@ -105,9 +107,6 @@ public bool UseDisassemblyDiagnoser [Option("monoPath", Required = false, HelpText = "Optional path to Mono which should be used for running benchmarks.")] public FileInfo? MonoPath { get; set; } - [Option("clrVersion", Required = false, HelpText = "Optional version of private CLR build used as the value of COMPLUS_Version env var.")] - public string? ClrVersion { get; set; } - [Option("ilCompilerVersion", Required = false, HelpText = "Optional version of Microsoft.DotNet.ILCompiler which should be used to run with NativeAOT. Example: \"7.0.0-preview.3.22123.2\"")] public string? ILCompilerVersion { get; set; } @@ -213,18 +212,12 @@ public bool UseDisassemblyDiagnoser [Option("wasmMainJsTemplate", Required = false, HelpText = "Path to main.mjs template.")] public FileInfo? WasmMainJsTemplate { get; set; } - [Option("customRuntimePack", Required = false, HelpText = "Path to a custom runtime pack. Only used for wasm/MonoAotLLVM currently.")] - public string? CustomRuntimePack { get; set; } + [Option("customRuntimePack", Required = false, HelpText = "Path to a custom runtime pack. Only used for ReadyToRun (R2R) currently.")] + public DirectoryInfo? CustomRuntimePack { get; set; } - [Option("AOTCompilerPath", Required = false, HelpText = "Path to Mono AOT compiler, used for MonoAotLLVM.")] + [Option("AOTCompilerPath", Required = false, HelpText = "Path to the crossgen2 compiler, used for ReadyToRun (R2R) benchmarks.")] public FileInfo? AOTCompilerPath { get; set; } - [Option("AOTCompilerMode", Required = false, Default = MonoAotCompilerMode.mini, HelpText = "Mono AOT compiler mode, either 'mini', 'llvm', or 'wasm'")] - public MonoAotCompilerMode AOTCompilerMode { get; set; } - - [Option("wasmRuntimeFlavor", Required = false, Default = Environments.RuntimeFlavor.Mono, HelpText = "Runtime flavor for WASM benchmarks: 'Mono' (default) uses the Mono runtime pack, 'CoreCLR' uses the CoreCLR runtime pack.")] - public Environments.RuntimeFlavor WasmRuntimeFlavor { get; set; } - [Option("wasmProcessTimeout", Required = false, Default = 10, HelpText = "Maximum time in minutes to wait for a single WASM benchmark process to finish before force killing it.")] public int WasmProcessTimeoutMinutes { get; set; } diff --git a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs index 3c15744439..d21da1710b 100644 --- a/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs +++ b/src/BenchmarkDotNet/ConsoleArguments/ConfigParser.cs @@ -13,13 +13,12 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CoreRun; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.Mono; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.Wasm; using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Toolchains.Framework; using BenchmarkDotNet.Toolchains.R2R; using CommandLine; using Perfolizer.Horology; @@ -28,6 +27,7 @@ using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Text; +using BenchmarkDotNet.Toolchains.NetCoreApp; namespace BenchmarkDotNet.ConsoleArguments { @@ -285,23 +285,13 @@ private static bool Validate(CommandLineOptions options, ILogger logger) foreach (string runtime in options.Runtimes) { - if (!TryParse(runtime, out RuntimeMoniker runtimeMoniker)) + if (!Runtime.TryParse(runtime, out _)) { - logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Available options are: {string.Join(", ", Enum.GetNames(typeof(RuntimeMoniker)).Select(name => name.ToLower()))}."); + logger.WriteLineError($"The provided runtime \"{runtime}\" is invalid. Expected one of:"); + foreach (string form in KnownRuntimeMonikerForms) + logger.WriteLineError($" {form}"); return false; } - else if (runtimeMoniker == RuntimeMoniker.MonoAOTLLVM && (options.AOTCompilerPath == null || options.AOTCompilerPath.IsNotNullButDoesNotExist())) - { - logger.WriteLineError($"The provided {nameof(options.AOTCompilerPath)} \"{options.AOTCompilerPath}\" does NOT exist. It MUST be provided."); - } - else if (runtimeMoniker >= RuntimeMoniker.WasmNet80 && runtimeMoniker < RuntimeMoniker.MonoAOTLLVM) - { - if (!ProcessHelper.TryResolveExecutableInPath(options.WasmJavaScriptEngine, out _)) - { - logger.WriteLineError($"The provided {nameof(options.WasmJavaScriptEngine)} \"{options.WasmJavaScriptEngine}\" does NOT exist."); - return false; - } - } } foreach (string exporter in options.Exporters) @@ -532,15 +522,9 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, { yield return Attributes.InProcessAttribute.GetJob(baseJob, Attributes.InProcessToolchainType.Auto, true); } - else if (options.ClrVersion.IsNotBlank()) - { - var runtime = ClrRuntime.CreateForLocalFullNetFrameworkBuild(options.ClrVersion); - yield return baseJob.WithRuntime(runtime).WithId(runtime.Name); // local builds of .NET Runtime - } - else if (options.CliPath != null && options.Runtimes.IsEmpty() && options.CoreRunPaths.IsEmpty()) - { - yield return CreateCoreJobWithCli(baseJob, options); - } + // --cli and --packages configure a toolchain without selecting one, so with no --runtimes or --corerun to + // attach them to they are ignored. Creating a job for the host runtime instead would either add a run + // nobody asked for or override the one the benchmark declares. else { // in case both --runtimes and --corerun are specified, the first one is returned first and becomes a baseline job @@ -571,224 +555,40 @@ private static IEnumerable Expand(Job baseJob, CommandLineOptions options, private static Job CreateJobForGivenRuntime(Job baseJob, string runtimeId, CommandLineOptions options) { - if (!TryParse(runtimeId, out RuntimeMoniker runtimeMoniker)) - { - throw new InvalidOperationException("Impossible, already validated by the Validate method"); - } + return Runtime.Parse(runtimeId) switch + { + ClrRuntime clr => GetFrameworkJob(clr).WithId(clr.ToString()), + CoreRuntime core => baseJob.WithId(core.ToString()) + .WithToolchain(CsProjCoreToolchain.From(core, new(options))), + NativeAotRuntime aot => baseJob.WithId(aot.ToString()) + .WithToolchain(CsProjNativeAotToolchain.From(aot, new(options))), + R2RRuntime r2r => baseJob.WithId(r2r.ToString()) + .WithToolchain(CsProjR2RToolchain.From(r2r, new(options))), + MonoWasmRuntime wasm => baseJob.WithId(wasm.ToString()) + .WithToolchain(CsProjMonoWasmToolchain.From(wasm, new(options))), + MonoWasmAotRuntime wasmAot => baseJob.WithId(wasmAot.ToString()) + .WithToolchain(CsProjMonoWasmAotToolchain.From(wasmAot, new(options))), + CoreWasmRuntime coreWasm => baseJob.WithId(coreWasm.ToString()) + .WithToolchain(CsProjCoreWasmToolchain.From(coreWasm, new(options))), + MonoCoreRuntime mono => baseJob.WithId(mono.ToString()) + .WithToolchain(CsProjMonoCoreToolchain.From(mono, new(options))), + MonoAotRuntime monoAot => baseJob.WithId(monoAot.ToString()) + .WithToolchain(RoslynMonoAotToolchain.From(new(options))), + MonoRuntime mono => baseJob.WithId(mono.ToString()) + .WithToolchain(RoslynMonoToolchain.From(new(options))), + _ => throw new NotSupportedException($"Runtime {runtimeId} is not supported"), + }; - switch (runtimeMoniker) + Job GetFrameworkJob(ClrRuntime clr) { - case RuntimeMoniker.Net461: - case RuntimeMoniker.Net462: - case RuntimeMoniker.Net47: - case RuntimeMoniker.Net471: - case RuntimeMoniker.Net472: - case RuntimeMoniker.Net48: - case RuntimeMoniker.Net481: - { - var runtime = runtimeMoniker.GetRuntime(); - return baseJob - .WithRuntime(runtime) - .WithId(runtime.Name) - .WithToolchain(CsProjClassicNetToolchain.From(runtimeId, options.RestorePath?.FullName ?? "", options.CliPath?.FullName ?? "")); - } - - case RuntimeMoniker.NetCoreApp20: - case RuntimeMoniker.NetCoreApp21: - case RuntimeMoniker.NetCoreApp22: - case RuntimeMoniker.NetCoreApp30: - case RuntimeMoniker.NetCoreApp31: - case RuntimeMoniker.Net50: - case RuntimeMoniker.Net60: - case RuntimeMoniker.Net70: - case RuntimeMoniker.Net80: - case RuntimeMoniker.Net90: - case RuntimeMoniker.Net10_0: - case RuntimeMoniker.Net11_0: - { - var runtime = runtimeMoniker.GetRuntime(); - return baseJob - .WithRuntime(runtime) - .WithId(runtime.Name) - .WithToolchain(CsProjCoreToolchain.From( - new NetCoreAppSettings( - runtimeId, - runtimeFrameworkVersion: "", - name: runtimeId, - options: options))); - } - - case RuntimeMoniker.Mono: - { - var runtime = new MonoRuntime("Mono", options.MonoPath?.FullName ?? ""); - return baseJob.WithRuntime(runtime).WithId(runtime.Name); - } - - case RuntimeMoniker.NativeAot70: - return CreateAotJob(baseJob, options, runtimeMoniker, ilCompilerVersion: ""); - - case RuntimeMoniker.NativeAot80: - return CreateAotJob(baseJob, options, runtimeMoniker, ilCompilerVersion: ""); - - case RuntimeMoniker.NativeAot90: - return CreateAotJob(baseJob, options, runtimeMoniker, ilCompilerVersion: ""); - - case RuntimeMoniker.NativeAot10_0: - return CreateAotJob(baseJob, options, runtimeMoniker, ilCompilerVersion: ""); - - case RuntimeMoniker.NativeAot11_0: - return CreateAotJob(baseJob, options, runtimeMoniker, ilCompilerVersion: ""); - - case RuntimeMoniker.WasmNet80: - return MakeWasmJob(baseJob, options, "net8.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet90: - return MakeWasmJob(baseJob, options, "net9.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet10_0: - return MakeWasmJob(baseJob, options, "net10.0", runtimeMoniker); - - case RuntimeMoniker.WasmNet11_0: - return MakeWasmJob(baseJob, options, "net11.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVM: - return MakeMonoAOTLLVMJob(baseJob, options, RuntimeInformation.IsNetCore ? CoreRuntime.GetCurrentVersion().MsBuildMoniker : "net6.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet60: - return MakeMonoAOTLLVMJob(baseJob, options, "net6.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet70: - return MakeMonoAOTLLVMJob(baseJob, options, "net7.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet80: - return MakeMonoAOTLLVMJob(baseJob, options, "net8.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet90: - return MakeMonoAOTLLVMJob(baseJob, options, "net9.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet10_0: - return MakeMonoAOTLLVMJob(baseJob, options, "net10.0", runtimeMoniker); - - case RuntimeMoniker.MonoAOTLLVMNet11_0: - return MakeMonoAOTLLVMJob(baseJob, options, "net11.0", runtimeMoniker); - - case RuntimeMoniker.Mono60: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono60); - - case RuntimeMoniker.Mono70: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono70); - - case RuntimeMoniker.Mono80: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono80); - - case RuntimeMoniker.Mono90: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono90); - - case RuntimeMoniker.Mono10_0: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono10_0); - - case RuntimeMoniker.Mono11_0: - return MakeMonoJob(baseJob, options, MonoRuntime.Mono11_0); - - case RuntimeMoniker.R2R80: - case RuntimeMoniker.R2R90: - case RuntimeMoniker.R2R10_0: - case RuntimeMoniker.R2R11_0: - return CreateR2RJob(baseJob, options, runtimeMoniker.GetRuntime()); - - default: - throw new NotSupportedException($"Runtime {runtimeId} is not supported"); + var settings = new FrameworkSettings(options); + return settings.Equals(FrameworkSettings.Default) + // If no custom settings were configured, we just set the runtime so the default toolchain will be auto-selected, which might select the faster Roslyn toolchain. + ? baseJob.WithRuntime(clr) + : baseJob.WithToolchain(CsProjFrameworkToolchain.From(clr, settings)); } } - private static Job CreateAotJob(Job baseJob, CommandLineOptions options, RuntimeMoniker runtimeMoniker, string ilCompilerVersion, string nuGetFeedUrl = "") - { - var builder = NativeAotToolchain.CreateBuilder(); - - if (options.CliPath != null) - builder.DotNetCli(options.CliPath.FullName); - if (options.RestorePath != null) - builder.PackagesRestorePath(options.RestorePath.FullName); - - if (options.IlcPackages != null) - builder.UseLocalBuild(options.IlcPackages); - else if (options.ILCompilerVersion.IsNotBlank()) - builder.UseNuGet(options.ILCompilerVersion, nuGetFeedUrl); - else - builder.UseNuGet(ilCompilerVersion, nuGetFeedUrl); - - var runtime = runtimeMoniker.GetRuntime(); - builder.TargetFrameworkMoniker(runtime.MsBuildMoniker); - - return baseJob.WithRuntime(runtime).WithToolchain(builder.ToToolchain()).WithId(runtime.Name); - } - - private static Job MakeMonoJob(Job baseJob, CommandLineOptions options, MonoRuntime runtime) - { - return baseJob - .WithRuntime(runtime) - .WithToolchain(MonoToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: runtime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: runtime.Name, - options: options))); - } - - private static Job MakeMonoAOTLLVMJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) - { - var monoAotLLVMRuntime = new MonoAotLLVMRuntime( - aotCompilerPath: options.AOTCompilerPath, - aotCompilerMode: options.AOTCompilerMode, - msBuildMoniker: msBuildMoniker, - moniker: moniker); - - var toolChain = MonoAotLLVMToolChain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: monoAotLLVMRuntime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: monoAotLLVMRuntime.Name, - options: options)); - - return baseJob.WithRuntime(monoAotLLVMRuntime).WithToolchain(toolChain).WithId(monoAotLLVMRuntime.Name); - } - - private static Job CreateR2RJob(Job baseJob, CommandLineOptions options, Runtime runtime) - { - var toolChain = R2RToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: runtime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: runtime.Name, - options: options)); - - return baseJob.WithRuntime(runtime).WithToolchain(toolChain).WithId(runtime.Name); - } - - private static Job MakeWasmJob(Job baseJob, CommandLineOptions options, string msBuildMoniker, RuntimeMoniker moniker) - { - bool wasmAot = options.AOTCompilerMode == MonoAotCompilerMode.wasm; - - var wasmRuntime = new WasmRuntime( - msBuildMoniker: msBuildMoniker, - moniker: moniker, - displayName: "Wasm", - javaScriptEngine: options.WasmJavaScriptEngine ?? "", - javaScriptEngineArguments: options.WasmJavaScriptEngineArguments, - aot: wasmAot, - runtimeFlavor: options.WasmRuntimeFlavor, - mainJsTemplate: options.WasmMainJsTemplate, - processTimeoutMinutes: options.WasmProcessTimeoutMinutes); - - var toolChain = WasmToolchain.From(new NetCoreAppSettings( - targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, - runtimeFrameworkVersion: "", - name: wasmRuntime.Name, - options: options)); - - return baseJob.WithRuntime(wasmRuntime).WithToolchain(toolChain).WithId(wasmRuntime.Name); - } - private static IEnumerable GetFilters(CommandLineOptions options) { if (options.Filters.Any()) @@ -818,25 +618,32 @@ private static int GetMaximumDisplayWidth() private static Job CreateCoreRunJob(Job baseJob, CommandLineOptions options, FileInfo coreRunPath) => baseJob - .WithToolchain(new CoreRunToolchain( - coreRunPath, - createCopy: true, - targetFrameworkMoniker: - RuntimeInformation.IsNetCore - ? RuntimeInformation.GetCurrentRuntime().MsBuildMoniker - : CoreRuntime.Latest.MsBuildMoniker, // use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors - customDotNetCliPath: options.CliPath, - restorePath: options.RestorePath, - displayName: GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath))); - - private static Job CreateCoreJobWithCli(Job baseJob, CommandLineOptions options) - => baseJob - .WithToolchain(CsProjCoreToolchain.From( - new NetCoreAppSettings( - targetFrameworkMoniker: RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, - runtimeFrameworkVersion: "", - name: RuntimeInformation.GetCurrentRuntime().Name, - options: options))); + .WithToolchain(CoreRunToolchain.From(new CoreRunSettings(options) + { + SourceCoreRun = coreRunPath, + TargetFrameworkMoniker = RuntimeInformation.GetCurrentRuntime() is CoreRuntime core + ? core.GetTfm() // netcoreappX.Y for < 5, netX.0 for 5+ + : CoreRuntime.Latest.GetTfm(), // non-Core host; use most recent tfm, as the toolchain is being used only by dotnet/runtime contributors + DisplayName = GetCoreRunToolchainDisplayName(options.CoreRunPaths, coreRunPath), + })); + + /// + /// The moniker forms accepts, for the "invalid runtime" error. Listed in the + /// same order as its prefix dispatch, so the two can be compared at a glance. + /// + private static readonly string[] KnownRuntimeMonikerForms = + [ + "net e.g. net472, net8.0, net8.0-windows (.NET 5.0+ only)", + "netcoreapp e.g. netcoreapp3.1", + "nativeaot e.g. nativeaot8.0", + "r2r e.g. r2r8.0", + "mono classic Mono", + "mono .NET on the Mono VM, e.g. mono8.0", + "monoaot legacy Mono AOT, versionless", + "monowasm e.g. monowasm8.0", + "monowasmaot e.g. monowasmaot8.0", + "corewasm e.g. corewasm11.0", + ]; /// /// we have a limited amount of space when printing the output to the console, so we try to keep things small and simple @@ -874,21 +681,5 @@ private static string GetCoreRunToolchainDisplayName(IReadOnlyList pat return coreRunPath.FullName.Substring(lastCommonDirectorySeparatorIndex); } - - internal static bool TryParse(string runtime, out RuntimeMoniker runtimeMoniker) - { - int index = runtime.IndexOf('-'); - if (index >= 0) - { - runtime = runtime.Substring(0, index); - } - - // Monikers older than Net 10 don't use any version delimiter, newer monikers use _ delimiter. - if (Enum.TryParse(runtime.Replace(".", string.Empty), ignoreCase: true, out runtimeMoniker)) - { - return true; - } - return Enum.TryParse(runtime.Replace('.', '_'), ignoreCase: true, out runtimeMoniker); - } } } diff --git a/src/BenchmarkDotNet/ConsoleArguments/RuntimeFlavor.cs b/src/BenchmarkDotNet/ConsoleArguments/RuntimeFlavor.cs deleted file mode 100644 index d4cd356781..0000000000 --- a/src/BenchmarkDotNet/ConsoleArguments/RuntimeFlavor.cs +++ /dev/null @@ -1,13 +0,0 @@ -namespace BenchmarkDotNet.Environments; - -/// -/// Specifies the .NET runtime flavor to use for WASM benchmarks. -/// -public enum RuntimeFlavor -{ - /// Uses the Mono runtime pack (Microsoft.NETCore.App.Runtime.Mono.browser-wasm). - Mono, - - /// Uses the CoreCLR runtime pack (Microsoft.NETCore.App.Runtime.browser-wasm). - CoreCLR -} diff --git a/src/BenchmarkDotNet/Detectors/OsDetector.cs b/src/BenchmarkDotNet/Detectors/OsDetector.cs index 2f360d01f4..4335953a06 100644 --- a/src/BenchmarkDotNet/Detectors/OsDetector.cs +++ b/src/BenchmarkDotNet/Detectors/OsDetector.cs @@ -170,6 +170,12 @@ internal static bool IsTvOS() => RuntimeInformation.IsOSPlatform(OSPlatform.Create("TVOS")); #endif + /// + /// Mobile OSes where BenchmarkDotNet can't spawn a child process to build/run benchmarks out of process, + /// so benchmarks have to be executed in-process regardless of the runtime. + /// + internal static bool IsMobile() => IsAndroid() || IsIOS() || IsTvOS(); + [SupportedOSPlatformGuard("windows6.1")] internal static bool IsWindows7OrLater() => #if NET6_0_OR_GREATER diff --git a/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs b/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs index 7d4b726233..c98f52e9ae 100644 --- a/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/EventPipeProfiler.cs @@ -4,7 +4,6 @@ using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; @@ -56,11 +55,20 @@ public async IAsyncEnumerable ValidateAsync(ValidationParameter { foreach (var benchmark in validationParameters.Benchmarks) { - var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance)!; + var runtime = benchmark.GetRuntime(); - if (runtime.RuntimeMoniker < RuntimeMoniker.NetCoreApp30) + // EventPipeProfiler is out-of-process: it attaches to the benchmark process by PID via the .NET + // diagnostics IPC server (DiagnosticsClient), introduced in .NET Core 3.0. WASM is excluded because it + // runs inside a JavaScript engine with no attachable diagnostics server. + bool supported = runtime switch { - yield return new ValidationError(true, $"{nameof(EventPipeProfiler)} supports only .NET Core 3.0+", benchmark); + CoreRuntime core => core.Version.Major >= 3, + NativeAotRuntime or R2RRuntime or MonoCoreRuntime => true, + _ => false, + }; + if (!supported) + { + yield return new ValidationError(true, $"{nameof(EventPipeProfiler)} supports only .NET (Core) 3.0+, but the job targets {runtime}", benchmark); } } } diff --git a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs index e0439dbe6f..e7fd051728 100644 --- a/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs +++ b/src/BenchmarkDotNet/Diagnosers/PerfCollectProfiler.cs @@ -10,9 +10,9 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CoreRun; -using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Toolchains.NetCoreApp; using BenchmarkDotNet.Toolchains.Results; using BenchmarkDotNet.Validators; using JetBrains.Annotations; @@ -207,14 +207,14 @@ private async ValueTask EnsureSymbolsForNativeRuntime(DiagnoserActionParameters return; // it's not needed for a local build of dotnet runtime } - string cliPath = parameters.BenchmarkCase.GetToolchain() switch + FileInfo? cliPath = parameters.BenchmarkCase.GetToolchain() switch { - CsProjCoreToolchain core => core.CustomDotNetCliPath, - NativeAotToolchain nativeAot => nativeAot.CustomDotNetCliPath, - _ => DotNetCliCommandExecutor.DefaultDotNetCliPath.Value + CsProjCoreToolchain core => core.Settings.CliPath, + CsProjNativeAotToolchain nativeAot => nativeAot.Settings.CliPath, + _ => null }; - if (!cliPathWithSymbolsInstalled.Add(cliPath)) + if (!cliPathWithSymbolsInstalled.Add(cliPath?.FullName ?? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value)) { return; } @@ -255,7 +255,7 @@ private async ValueTask EnsureSymbolsForNativeRuntime(DiagnoserActionParameters } await DotNetCliCommandExecutor.ExecuteAsync(cliCommand - .WithCliPath(Path.Combine(toolPath, "dotnet-symbol")) + .WithCliPath(new FileInfo(Path.Combine(toolPath, "dotnet-symbol"))) .WithArguments($"--recurse-subdirectories --symbols \"{dotnetPath}/dotnet\" \"{dotnetPath}/lib*.so\""), cancellationToken) .ConfigureAwait(false); diff --git a/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs index a95a718e64..d49569aa32 100644 --- a/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs +++ b/src/BenchmarkDotNet/Diagnosers/SnapshotProfilerBase.cs @@ -1,7 +1,7 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; @@ -22,7 +22,7 @@ public abstract class SnapshotProfilerBase : IProfiler protected abstract string CreateSnapshotFilePath(DiagnoserActionParameters parameters); protected abstract string GetRunnerPath(); - internal abstract bool IsSupported(RuntimeMoniker runtimeMoniker); + internal abstract bool IsSupported(Runtime runtime); private readonly List snapshotFilePaths = []; @@ -31,7 +31,7 @@ public abstract class SnapshotProfilerBase : IProfiler public IEnumerable Analysers => []; public RunMode GetRunMode(BenchmarkCase benchmarkCase) => - IsSupported(benchmarkCase.Job.Environment.GetRuntime().RuntimeMoniker) ? RunMode.ExtraRun : RunMode.None; + IsSupported(benchmarkCase.GetRuntime()) ? RunMode.ExtraRun : RunMode.None; public ValueTask HandleAsync(HostSignal signal, DiagnoserActionParameters parameters, CancellationToken cancellationToken) @@ -39,10 +39,10 @@ public ValueTask HandleAsync(HostSignal signal, DiagnoserActionParameters parame var logger = parameters.Config.GetCompositeLogger(); var job = parameters.BenchmarkCase.Job; - var runtimeMoniker = job.Environment.GetRuntime().RuntimeMoniker; - if (!IsSupported(runtimeMoniker)) + var runtime = parameters.BenchmarkCase.GetRuntime(); + if (!IsSupported(runtime)) { - logger.WriteLineError($"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + logger.WriteLineError($"Runtime '{runtime}' is not supported by dotMemory"); return new(); } @@ -64,10 +64,10 @@ public ValueTask HandleAsync(HostSignal signal, DiagnoserActionParameters parame public async IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) { - var runtimeMonikers = validationParameters.Benchmarks.Select(b => b.Job.Environment.GetRuntime().RuntimeMoniker).Distinct(); - foreach (var runtimeMoniker in runtimeMonikers) - if (!IsSupported(runtimeMoniker)) - yield return new ValidationError(true, $"Runtime '{runtimeMoniker}' is not supported by dotMemory"); + var runtimes = validationParameters.Benchmarks.Select(b => b.GetRuntime()).Distinct(); + foreach (var runtime in runtimes) + if (!IsSupported(runtime)) + yield return new ValidationError(true, $"Runtime '{runtime}' is not supported by {ShortName}"); } public IEnumerable ProcessResults(DiagnoserResults results) => []; diff --git a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs index 4c4dca8b43..a5613fb805 100644 --- a/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs +++ b/src/BenchmarkDotNet/Diagnosers/ThreadingDiagnoser.cs @@ -3,7 +3,6 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; @@ -50,11 +49,16 @@ public async IAsyncEnumerable ValidateAsync(ValidationParameter { foreach (var benchmark in validationParameters.Benchmarks) { - var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance); - - if (runtime != null && runtime.RuntimeMoniker < RuntimeMoniker.NetCoreApp30) + var runtime = benchmark.GetRuntime(); + bool supported = runtime switch + { + CoreRuntime core => core.Version.Major >= 3, + NativeAotRuntime or R2RRuntime or MonoCoreRuntime or WasmRuntime => true, + _ => false, + }; + if (!supported) { - yield return new ValidationError(true, $"{nameof(ThreadingDiagnoser)} supports only .NET Core 3.0+", benchmark); + yield return new ValidationError(true, $"{nameof(ThreadingDiagnoser)} supports only .NET (Core) 3.0+, but the job targets {runtime}", benchmark); } } } diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdArgs.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdArgs.cs index fadec8ca18..41faf28111 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdArgs.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdArgs.cs @@ -2,7 +2,7 @@ namespace BenchmarkDotNet.Disassemblers { - internal struct ClrMdArgs(int processId, string typeName, string methodName, bool printSource, int maxDepth, string syntax, string tfm, string[] filters, string resultsPath = "") + internal struct ClrMdArgs(int processId, string typeName, string methodName, bool printSource, int maxDepth, string syntax, Version runtimeVersion, string[] filters, string resultsPath = "") { [JsonIgnore] internal int ProcessId = processId; @@ -26,7 +26,7 @@ internal struct ClrMdArgs(int processId, string typeName, string methodName, boo internal string Syntax = syntax; [JsonInclude] - internal string TargetFrameworkMoniker = tfm; + internal Version RuntimeVersion = runtimeVersion; [JsonInclude] internal string ResultsPath = resultsPath; @@ -40,8 +40,8 @@ internal static ClrMdArgs FromArgs(string[] args) maxDepth: int.Parse(args[4]), resultsPath: args[5], syntax: args[6], - tfm: args[7], + runtimeVersion: Version.Parse(args[7]), filters: [.. args.Skip(8)] ); } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Disassemblers/ClrMdDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/ClrMdDisassembler.cs index cc54ddea0c..580fd82afb 100644 --- a/src/BenchmarkDotNet/Disassemblers/ClrMdDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/ClrMdDisassembler.cs @@ -74,7 +74,7 @@ internal DisassemblyResult AttachAndDisassemble(ClrMdArgs args) var runtime = dataTarget.ClrVersions.Single().CreateRuntime(); - var state = new State(runtime, args.TargetFrameworkMoniker); + var state = new State(runtime, args.RuntimeVersion); if (args.Filters.Length > 0) { diff --git a/src/BenchmarkDotNet/Disassemblers/DataContracts.cs b/src/BenchmarkDotNet/Disassemblers/DataContracts.cs index c7b5ddf428..e3917ce21a 100644 --- a/src/BenchmarkDotNet/Disassemblers/DataContracts.cs +++ b/src/BenchmarkDotNet/Disassemblers/DataContracts.cs @@ -218,13 +218,13 @@ public static class DisassemblerConstants internal sealed class State { - internal State(ClrRuntime runtime, string targetFrameworkMoniker) + internal State(ClrRuntime runtime, Version runtimeVersion) { Runtime = runtime; Todo = new Queue(); HandledMethods = new HashSet(new ClrMethodComparer()); AddressToNameMapping = []; - RuntimeVersion = ParseVersion(targetFrameworkMoniker); + RuntimeVersion = runtimeVersion; } internal ClrRuntime Runtime { get; } @@ -233,31 +233,6 @@ internal State(ClrRuntime runtime, string targetFrameworkMoniker) internal Dictionary AddressToNameMapping { get; } internal Version RuntimeVersion { get; } - internal static Version ParseVersion(string targetFrameworkMoniker) - { - int firstDigit = -1, lastDigit = -1; - for (int i = 0; i < targetFrameworkMoniker.Length; i++) - { - if (char.IsDigit(targetFrameworkMoniker[i])) - { - if (firstDigit == -1) - firstDigit = i; - - lastDigit = i; - } - else if (targetFrameworkMoniker[i] == '-') - { - break; // it can be platform specific like net7.0-windows8 - } - } - - string versionToParse = targetFrameworkMoniker.Substring(firstDigit, lastDigit - firstDigit + 1); - if (!versionToParse.Contains(".")) // Full .NET Framework (net48 etc) - versionToParse = string.Join(".", versionToParse.ToCharArray()); - - return Version.Parse(versionToParse); - } - private sealed class ClrMethodComparer : IEqualityComparer { public bool Equals(ClrMethod? x, ClrMethod? y) diff --git a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs index 9ca2cbda84..cb4412ad08 100644 --- a/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs +++ b/src/BenchmarkDotNet/Disassemblers/DisassemblyDiagnoser.cs @@ -7,7 +7,6 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Reports; @@ -84,7 +83,7 @@ private ClrMdArgs BuildClrMdArgs(BenchmarkCase benchmarkCase, string typeName, i maxDepth: Config.MaxDepth, filters: Config.Filters, syntax: Config.Syntax.ToString(), - tfm: benchmarkCase.Job.Environment.GetRuntime().MsBuildMoniker + runtimeVersion: benchmarkCase.GetRuntime().Version! ); public async ValueTask HandleAsync(HostSignal signal, DiagnoserActionParameters parameters, CancellationToken cancellationToken) @@ -100,7 +99,7 @@ public async ValueTask HandleAsync(HostSignal signal, DiagnoserActionParameters ); break; case HostSignal.SeparateLogic when ShouldUseMonoDisassembler(benchmark): - var result = await monoDisassembler.Disassemble(benchmark, (MonoRuntime)benchmark.Job.Environment.Runtime!, cancellationToken).ConfigureAwait(false); + var result = await monoDisassembler.Disassemble(benchmark, cancellationToken).ConfigureAwait(false); results.Add(benchmark, result); break; } @@ -117,13 +116,13 @@ public async IAsyncEnumerable ValidateAsync(ValidationParameter var currentPlatform = RuntimeInformation.GetCurrentPlatform(); if (!(currentPlatform is Platform.X64 or Platform.X86 or Platform.Arm64)) { - yield return new ValidationError(true, $"DisassemblyDiagnoser does not support {currentPlatform}"); + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} does not support {currentPlatform}"); yield break; } if (currentPlatform == Platform.Arm64 && OsDetector.IsWindows()) { - yield return new ValidationError(true, $"DisassemblyDiagnoser does not support Arm on Windows"); + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} does not support Arm on Windows"); yield break; } @@ -135,29 +134,27 @@ public async IAsyncEnumerable ValidateAsync(ValidationParameter foreach (var benchmark in validationParameters.Benchmarks) { + var runtime = benchmark.GetRuntime(); if (benchmark.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is InProcessNoEmitToolchain) { - yield return new ValidationError(true, "InProcessToolchain has no DisassemblyDiagnoser support", benchmark); + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} does not support {nameof(InProcessNoEmitToolchain)}", benchmark); } - else if (benchmark.Job.IsNativeAOT()) + else if (runtime is not (CoreRuntime or R2RRuntime or ClrRuntime or LegacyMonoRuntime)) { - yield return new ValidationError(true, "Currently NativeAOT has no DisassemblyDiagnoser support", benchmark); + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} does not support {runtime}", benchmark); } - - if (ShouldUseClrMdDisassembler(benchmark)) + else if (ShouldUseClrMdDisassembler(benchmark)) { if (Config.RunInHost && toolchain?.IsInProcess != true && !PlatformsMatch(currentPlatform, benchmark.Job.Environment.Platform)) { - yield return new ValidationError(true, "DisassemblyDiagnoser cannot run in host for a job that targets a different platform", benchmark); + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} cannot run in host for a job that targets a different platform", benchmark); } if (OsDetector.IsLinux()) { - var runtime = benchmark.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance)!; - - if (runtime.RuntimeMoniker < RuntimeMoniker.NetCoreApp30) + if (runtime is CoreRuntime core && core.Version.Major < 3 && !Config.RunInHost) { - yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} supports only .NET Core 3.0+", benchmark); + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} supports only .NET (Core) 3.0+ with RunInHost option, but the job targets {runtime}", benchmark); } if (ptrace_scope.Value == "2") @@ -172,7 +169,7 @@ public async IAsyncEnumerable ValidateAsync(ValidationParameter } else if (!ShouldUseMonoDisassembler(benchmark)) { - yield return new ValidationError(true, $"Only Windows and Linux are supported in DisassemblyDiagnoser without Mono. Current OS is {System.Runtime.InteropServices.RuntimeInformation.OSDescription}"); + yield return new ValidationError(true, $"{nameof(DisassemblyDiagnoser)} does not support this OS ({System.Runtime.InteropServices.RuntimeInformation.OSDescription}) without legacy Mono"); } } } @@ -190,8 +187,7 @@ private static bool PlatformsMatch(Platform currentPlatform, Platform targetPlat } private static bool ShouldUseMonoDisassembler(BenchmarkCase benchmarkCase) - => benchmarkCase.Job.Environment.Runtime is MonoRuntime - || (RuntimeInformation.IsMono && benchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain.IsInProcess); + => benchmarkCase.GetRuntime() is LegacyMonoRuntime; private static bool ShouldUseClrMdDisassembler(BenchmarkCase benchmarkCase) => !ShouldUseMonoDisassembler(benchmarkCase) && (OsDetector.IsWindows() || OsDetector.IsLinux() || OsDetector.IsMacOS()); diff --git a/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs b/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs index df264877c1..ec5caea8ed 100644 --- a/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs +++ b/src/BenchmarkDotNet/Disassemblers/MonoDisassembler.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Mono; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Reflection; @@ -13,7 +14,7 @@ namespace BenchmarkDotNet.Disassemblers { internal sealed class MonoDisassembler { - internal async Task Disassemble(BenchmarkCase benchmarkCase, MonoRuntime mono, CancellationToken cancellationToken) + internal async Task Disassemble(BenchmarkCase benchmarkCase, CancellationToken cancellationToken) { Debug.Assert(!RuntimeInformation.IsMono, "Must never be called for Non-Mono benchmarks"); @@ -23,7 +24,9 @@ internal async Task Disassemble(BenchmarkCase benchmarkCase, string exePath = benchmarkTarget.Type.GetTypeInfo().Assembly.Location; var environmentVariables = new Dictionary { ["MONO_VERBOSE_METHOD"] = fqnMethod }; - string monoPath = mono.CustomPath.IsNotBlank() ? mono.CustomPath : "mono"; + string monoPath = benchmarkCase.GetToolchain().Executor is LegacyMonoExecutor monoExecutor + ? monoExecutor.Settings.MonoPath?.FullName ?? "mono" + : "mono"; string arguments = $"--compile {fqnMethod} {llvmFlag} {exePath}"; var (_, output) = await ProcessHelper.RunAndReadOutputLineByLineAsync(monoPath, arguments, environmentVariables: environmentVariables, includeErrors: true, cancellationToken: cancellationToken) diff --git a/src/BenchmarkDotNet/Engines/GcStats.cs b/src/BenchmarkDotNet/Engines/GcStats.cs index c196d0b560..e1fef3b0e0 100644 --- a/src/BenchmarkDotNet/Engines/GcStats.cs +++ b/src/BenchmarkDotNet/Engines/GcStats.cs @@ -1,5 +1,5 @@ using BenchmarkDotNet.Columns; -using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using JetBrains.Annotations; @@ -39,7 +39,12 @@ private GcStats(int gen0Collections, int gen1Collections, int gen2Collections, l public long? GetBytesAllocatedPerOperation(BenchmarkCase benchmarkCase) { - bool excludeAllocationQuantumSideEffects = benchmarkCase.GetRuntime().RuntimeMoniker <= RuntimeMoniker.NetCoreApp20; // the issue got fixed for .NET Core 2.0+ https://github.com/dotnet/coreclr/issues/10207 + // .NET Framework (AppDomain.MonitoringTotalAllocatedMemorySize) and legacy Mono measure allocations at + // allocation-quantum granularity, and .NET Core <= 2.0 has the same quantum side effect (fixed in 2.1+, + // https://github.com/dotnet/coreclr/issues/10207). Matches the pre-refactor RuntimeMoniker <= NetCoreApp20 check. + var runtime = benchmarkCase.GetRuntime(); + bool excludeAllocationQuantumSideEffects = runtime is ClrRuntime or LegacyMonoRuntime + || (runtime is CoreRuntime core && core.Version <= new Version(2, 0)); long? allocatedBytes = GetTotalAllocatedBytes(excludeAllocationQuantumSideEffects); return allocatedBytes == null ? null diff --git a/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs b/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs index 672041ab05..4292f4fd83 100644 --- a/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs +++ b/src/BenchmarkDotNet/Environments/EnvironmentResolver.cs @@ -15,7 +15,6 @@ public class EnvironmentResolver : Resolver private EnvironmentResolver() { Register(EnvironmentMode.PlatformCharacteristic, RuntimeInformation.GetCurrentPlatform); - Register(EnvironmentMode.RuntimeCharacteristic, RuntimeInformation.GetCurrentRuntime); Register(EnvironmentMode.JitCharacteristic, JitInfo.GetCurrentJit); Register(EnvironmentMode.AffinityCharacteristic, RuntimeInformation.GetCurrentAffinity); Register(EnvironmentMode.EnvironmentVariablesCharacteristic, Array.Empty); diff --git a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs index 0a46673840..3d4987c6ea 100644 --- a/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs +++ b/src/BenchmarkDotNet/Environments/InfrastructureResolver.cs @@ -1,6 +1,7 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; using Perfolizer.Horology; namespace BenchmarkDotNet.Environments @@ -11,6 +12,7 @@ public class InfrastructureResolver : Resolver private InfrastructureResolver() { + Register(InfrastructureMode.RuntimeCharacteristic, RuntimeInformation.GetCurrentRuntime); Register(InfrastructureMode.ClockCharacteristic, () => Chronometer.BestClock); Register(InfrastructureMode.EngineFactoryCharacteristic, () => new EngineFactory()); Register(InfrastructureMode.BuildConfigurationCharacteristic, () => InfrastructureMode.ReleaseConfigurationName); diff --git a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs index 1b316fff67..dc35bd63c1 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/ClrRuntime.cs @@ -1,46 +1,28 @@ using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Framework; using System.Reflection; namespace BenchmarkDotNet.Environments { - public class ClrRuntime : Runtime, IEquatable + public sealed class ClrRuntime : Runtime { - public static readonly ClrRuntime Net461 = new ClrRuntime(RuntimeMoniker.Net461, "net461", ".NET Framework 4.6.1"); - public static readonly ClrRuntime Net462 = new ClrRuntime(RuntimeMoniker.Net462, "net462", ".NET Framework 4.6.2"); - public static readonly ClrRuntime Net47 = new ClrRuntime(RuntimeMoniker.Net47, "net47", ".NET Framework 4.7"); - public static readonly ClrRuntime Net471 = new ClrRuntime(RuntimeMoniker.Net471, "net471", ".NET Framework 4.7.1"); - public static readonly ClrRuntime Net472 = new ClrRuntime(RuntimeMoniker.Net472, "net472", ".NET Framework 4.7.2"); - public static readonly ClrRuntime Net48 = new ClrRuntime(RuntimeMoniker.Net48, "net48", ".NET Framework 4.8"); - public static readonly ClrRuntime Net481 = new ClrRuntime(RuntimeMoniker.Net481, "net481", ".NET Framework 4.8.1"); + public static readonly ClrRuntime Net461 = new(new(4, 6, 1)); + public static readonly ClrRuntime Net462 = new(new(4, 6, 2)); + public static readonly ClrRuntime Net47 = new(new(4, 7)); + public static readonly ClrRuntime Net471 = new(new(4, 7, 1)); + public static readonly ClrRuntime Net472 = new(new(4, 7, 2)); + public static readonly ClrRuntime Net48 = new(new(4, 8)); + public static readonly ClrRuntime Net481 = new(new(4, 8, 1)); - public string Version { get; } + public override string Name => ".NET Framework"; + public override Version Version { get; } - private ClrRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName, string version = "") - : base(runtimeMoniker, msBuildMoniker, displayName) - { - Version = version; - } - - /// YOU PROBABLY DON'T NEED IT, but if you are a .NET Runtime developer.. - /// please set it to particular .NET Runtime version if you want to benchmark it. - /// BenchmarkDotNet in going to pass `COMPLUS_Version` env var to the process for you. - /// - public static ClrRuntime CreateForLocalFullNetFrameworkBuild(string version) - { - if (string.IsNullOrEmpty(version)) throw new ArgumentNullException(nameof(version)); - - var current = GetCurrentVersion(); - - return new ClrRuntime(current.RuntimeMoniker, current.MsBuildMoniker, version, version); - } - - public override bool Equals(object? obj) => obj is ClrRuntime other && Equals(other); - - public bool Equals(ClrRuntime? other) => other != null && base.Equals(other) && Version == other.Version; - - public override int GetHashCode() => HashCode.Combine(base.GetHashCode(), Version); + private ClrRuntime(Version version) => Version = ToRuntimeVersion(version, keepBuild: true); internal static ClrRuntime GetCurrentVersion() { @@ -49,9 +31,9 @@ internal static ClrRuntime GetCurrentVersion() throw new PlatformNotSupportedException(".NET Framework supports Windows OS only."); } - string version = FrameworkVersionHelper.GetLatestNetDeveloperPackVersion() + var version = FrameworkVersionHelper.GetLatestNetDeveloperPackVersion() ?? FrameworkVersionHelper.GetFrameworkReleaseVersion(); // .NET Developer Pack is not installed - return GetRuntimeFromVersion(version); + return FromVersion(version); } internal static ClrRuntime GetTargetOrCurrentVersion(Assembly? assembly) @@ -62,25 +44,35 @@ internal static ClrRuntime GetTargetOrCurrentVersion(Assembly? assembly) } // Try to determine the Framework version that the assembly was compiled for. - string? version = FrameworkVersionHelper.GetTargetFrameworkVersion(assembly); + var version = FrameworkVersionHelper.GetTargetFrameworkVersion(assembly); return version != null - ? GetRuntimeFromVersion(version) + ? FromVersion(version) // Fallback to the current running Framework version. : GetCurrentVersion(); } - private static ClrRuntime GetRuntimeFromVersion(string version) - => version switch + internal static ClrRuntime FromVersion(Version version) + => (version.Major, version.Minor, version.Build) switch { - "4.6.1" => Net461, - "4.6.2" => Net462, - "4.7" => Net47, - "4.7.1" => Net471, - "4.7.2" => Net472, - "4.8" => Net48, - "4.8.1" => Net481, + (4, 8, 1) => Net481, + (4, 8, _) => Net48, + (4, 7, 2) => Net472, + (4, 7, 1) => Net471, + (4, 7, _) => Net47, + (4, 6, 2) => Net462, + (4, 6, 1) => Net461, // unlikely to happen but theoretically possible - _ => new ClrRuntime(RuntimeMoniker.NotRecognized, $"net{version.Replace(".", null)}", $".NET Framework {version}"), + _ => new ClrRuntime(version), }; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + { + if (!benchmarkCase.Job.HasDynamicBuildCharacteristic() && RuntimeInformation.IsFullFramework && Equals(GetTargetOrCurrentVersion(benchmarkCase.Descriptor.Type.Assembly))) + // The in-place Roslyn toolchain runs the already-loaded assembly. When the requested version differs + // from the running framework, report the requested one so the job and summary stay consistent. + return RoslynFrameworkToolchain.From(this); + + return CsProjFrameworkToolchain.From(this, FrameworkSettings.Default); + } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs index 9eed15bd41..6bac4c19fe 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreRuntime.cs @@ -1,7 +1,10 @@ using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.NetCoreApp; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Reflection; @@ -9,44 +12,122 @@ namespace BenchmarkDotNet.Environments { - public class CoreRuntime : Runtime + /// + /// Represents a specific version of the .NET (Core) runtime. + /// + public sealed class CoreRuntime : Runtime { - public static readonly CoreRuntime Core20 = new(RuntimeMoniker.NetCoreApp20, "netcoreapp2.0", ".NET Core 2.0"); - public static readonly CoreRuntime Core21 = new(RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", ".NET Core 2.1"); - public static readonly CoreRuntime Core22 = new(RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", ".NET Core 2.2"); - public static readonly CoreRuntime Core30 = new(RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", ".NET Core 3.0"); - public static readonly CoreRuntime Core31 = new(RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", ".NET Core 3.1"); - public static readonly CoreRuntime Core50 = new(RuntimeMoniker.Net50, "net5.0", ".NET 5.0"); - public static readonly CoreRuntime Core60 = new(RuntimeMoniker.Net60, "net6.0", ".NET 6.0"); - public static readonly CoreRuntime Core70 = new(RuntimeMoniker.Net70, "net7.0", ".NET 7.0"); - public static readonly CoreRuntime Core80 = new(RuntimeMoniker.Net80, "net8.0", ".NET 8.0"); - public static readonly CoreRuntime Core90 = new(RuntimeMoniker.Net90, "net9.0", ".NET 9.0"); - public static readonly CoreRuntime Core10_0 = new(RuntimeMoniker.Net10_0, "net10.0", ".NET 10.0"); - public static readonly CoreRuntime Core11_0 = new(RuntimeMoniker.Net11_0, "net11.0", ".NET 11.0"); + public static readonly CoreRuntime Core20 = new(new(2, 0)); + public static readonly CoreRuntime Core21 = new(new(2, 1)); + public static readonly CoreRuntime Core22 = new(new(2, 2)); + public static readonly CoreRuntime Core30 = new(new(3, 0)); + public static readonly CoreRuntime Core31 = new(new(3, 1)); + public static readonly CoreRuntime Core50 = new(new(5, 0)); + public static readonly CoreRuntime Core60 = new(new(6, 0)); + public static readonly CoreRuntime Core70 = new(new(7, 0)); + public static readonly CoreRuntime Core80 = new(new(8, 0)); + public static readonly CoreRuntime Core90 = new(new(9, 0)); + public static readonly CoreRuntime Core10_0 = new(new(10, 0)); + public static readonly CoreRuntime Core11_0 = new(new(11, 0)); public static CoreRuntime Latest => Core11_0; // when dotnet/runtime branches for 12.0, this will need to get updated - private CoreRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) - : base(runtimeMoniker, msBuildMoniker, displayName) + private readonly string? platform; + + private CoreRuntime(Version version, string? platform = null) { + Version = ToRuntimeVersion(version); + this.platform = platform; + Name = version.Major < 5 ? ".NET Core" : ".NET"; } - public bool IsPlatformSpecific => MsBuildMoniker.IndexOf('-') > 0; + public override string Name { get; } + + public override Version Version { get; } + + public bool IsPlatformSpecific => platform.IsNotBlank(); + + public string? Platform => platform; + + // The base compares Name and Version, which no longer carry the platform, so a net8.0-windows job would + // otherwise deduplicate against a plain net8.0 one. Compared as given, like everywhere else the platform is + // used: "net8.0-Windows" and "net8.0-windows" are two runtimes, and deduplicating them is the caller's call. + public override bool Equals(object? obj) + => base.Equals(obj) && platform == ((CoreRuntime) obj!).platform; + + public override int GetHashCode() + => HashCode.Combine(base.GetHashCode(), platform); + + /// Appends the target platform, when there is one, so platform-specific jobs are distinguishable. + public override string ToString() => IsPlatformSpecific ? $"{base.ToString()} ({platform})" : base.ToString(); /// - /// use this method if you want to target .NET version not supported by current version of BenchmarkDotNet. Example: .NET 10 + /// Whether the string is shaped like a target platform: a name, optionally followed by a version + /// ("windows", "windows10.0.19041.0"). /// - /// msbuild moniker, example: net10.0 - /// display name used by BDN to print the results - /// new runtime information - public static CoreRuntime CreateForNewVersion(string msBuildMoniker, string displayName) + /// + /// It ends up verbatim in the generated project's TargetFrameworks, where a stray '<' would produce + /// malformed XML and a ';' a second target framework. The shape is checked rather than the value matched + /// against known platforms, which would need updating for every new one. + /// + internal static bool IsValidPlatform(string platform) { - if (string.IsNullOrEmpty(msBuildMoniker)) throw new ArgumentNullException(nameof(msBuildMoniker)); - if (string.IsNullOrEmpty(displayName)) throw new ArgumentNullException(nameof(displayName)); + int index = 0; + while (index < platform.Length && char.IsLetter(platform[index])) + index++; + + if (index == 0) // has to start with a platform name + return false; + + // Optionally followed by a version. Every dot has to separate two digits - a leading, trailing or doubled + // one produces a moniker MSBuild rejects. + for (; index < platform.Length; index++) + { + if (platform[index] == '.') + { + bool separatesDigits = index > 0 && char.IsDigit(platform[index - 1]) + && index + 1 < platform.Length && char.IsDigit(platform[index + 1]); + if (!separatesDigits) + return false; + } + else if (!char.IsDigit(platform[index])) + { + return false; + } + } - return new CoreRuntime(RuntimeMoniker.NotRecognized, msBuildMoniker, displayName); + return true; } + /// Returns a runtime for the given version and optional platform. + /// + /// The platform is not shaped like a target platform identifier, or the version predates platform-specific + /// monikers, which would leave the platform in the runtime but absent from the moniker built from it. + /// + public static CoreRuntime From(Version version, string? platform = null) + => platform.IsNotBlank() + ? version.Major >= 5 && IsValidPlatform(platform!) + ? new CoreRuntime(version, platform) + : throw new ArgumentException( + $"'{platform}' is not a valid target platform for .NET {version.Major}.{version.Minor}. It has to be a platform name, optionally followed by a version, for example \"windows\" or \"windows10.0.19041.0\", and only .NET 5.0 and later have platform-specific target frameworks.", + nameof(platform)) + : (version.Major, version.Minor) switch + { + (2, 0) => Core20, + (2, 1) => Core21, + (2, 2) => Core22, + (3, 0) => Core30, + (3, 1) => Core31, + (5, 0) => Core50, + (6, 0) => Core60, + (7, 0) => Core70, + (8, 0) => Core80, + (9, 0) => Core90, + (10, 0) => Core10_0, + (11, 0) => Core11_0, + _ => new CoreRuntime(version), + }; + internal static CoreRuntime GetTargetOrCurrentVersion(Assembly? assembly) // Try to determine the version that the assembly was compiled for. => FrameworkVersionHelper.GetTargetCoreVersion(assembly) is { } version @@ -66,25 +147,19 @@ internal static CoreRuntime GetCurrentVersion() throw new NotSupportedException("Unable to recognize .NET Core version, please report a bug at https://github.com/dotnet/BenchmarkDotNet"); } - return FromVersion(version, null); + return FromVersion(version, Assembly.GetEntryAssembly()); } - internal static CoreRuntime FromVersion(Version version, Assembly? assembly = null) => version switch - { - { Major: 2, Minor: 0 } => Core20, - { Major: 2, Minor: 1 } => Core21, - { Major: 2, Minor: 2 } => Core22, - { Major: 3, Minor: 0 } => Core30, - { Major: 3, Minor: 1 } => Core31, - { Major: 5 } => GetPlatformSpecific(Core50, assembly), - { Major: 6 } => GetPlatformSpecific(Core60, assembly), - { Major: 7 } => GetPlatformSpecific(Core70, assembly), - { Major: 8 } => GetPlatformSpecific(Core80, assembly), - { Major: 9 } => GetPlatformSpecific(Core90, assembly), - { Major: 10 } => GetPlatformSpecific(Core10_0, assembly), - { Major: 11 } => GetPlatformSpecific(Core11_0, assembly), - _ => CreateForNewVersion($"net{version.Major}.{version.Minor}", $".NET {version.Major}.{version.Minor}"), - }; + private static CoreRuntime FromVersion(Version version, Assembly? assembly) + => (version.Major, version.Minor) switch + { + (2, 0) => Core20, + (2, 1) => Core21, + (2, 2) => Core22, + (3, 0) => Core30, + (3, 1) => Core31, + _ => GetPlatformSpecific(version, assembly), + }; internal static bool TryGetVersion([NotNullWhen(true)] out Version? version) { @@ -153,7 +228,7 @@ internal static string GetVersionFromFrameworkDescription() // .NET 10.0.0-preview.5.25277.114 -> 10.0.0-preview.5.25277.114 // .NET Core 3.1.32 -> 3.1.32 string frameworkDescription = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription; - return new string(frameworkDescription.SkipWhile(c => !char.IsDigit(c)).ToArray()); + return new([.. frameworkDescription.SkipWhile(c => !char.IsDigit(c))]); } // sample input: @@ -180,7 +255,7 @@ internal static bool TryGetVersionFromProductInfo(string productVersion, string { if (productVersion.IsNotBlank() && productName.IsNotBlank()) { - if (productName.IndexOf(".NET Core", StringComparison.OrdinalIgnoreCase) >= 0) + if (productName.Contains(".NET Core", StringComparison.OrdinalIgnoreCase)) { string parsableVersion = GetParsableVersionPart(productVersion); if (Version.TryParse(productVersion, out version) || Version.TryParse(parsableVersion, out version)) @@ -190,13 +265,13 @@ internal static bool TryGetVersionFromProductInfo(string productVersion, string } // yes, .NET Core 2.X has a product name == .NET Framework... - if (productName.IndexOf(".NET Framework", StringComparison.OrdinalIgnoreCase) >= 0) + if (productName.Contains(".NET Framework", StringComparison.OrdinalIgnoreCase)) { const string releaseVersionPrefix = "release/"; int releaseVersionIndex = productVersion.IndexOf(releaseVersionPrefix, StringComparison.Ordinal); if (releaseVersionIndex > 0) { - string releaseVersion = GetParsableVersionPart(productVersion.Substring(releaseVersionIndex + releaseVersionPrefix.Length)); + string releaseVersion = GetParsableVersionPart(productVersion[(releaseVersionIndex + releaseVersionPrefix.Length)..]); return Version.TryParse(releaseVersion, out version); } @@ -215,7 +290,7 @@ internal static bool TryGetVersionFromFrameworkName(string frameworkName, [NotNu const string versionPrefix = ".NETCoreApp,Version=v"; if (frameworkName.IsNotBlank() && frameworkName.StartsWith(versionPrefix)) { - string frameworkVersion = GetParsableVersionPart(frameworkName.Substring(versionPrefix.Length)); + string frameworkVersion = GetParsableVersionPart(frameworkName[versionPrefix.Length..]); return Version.TryParse(frameworkVersion, out version); } @@ -225,12 +300,22 @@ internal static bool TryGetVersionFromFrameworkName(string frameworkName, [NotNu } // Version.TryParse does not handle thing like 3.0.0-WORD - internal static string GetParsableVersionPart(string fullVersionName) => new string(fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.').ToArray()); + internal static string GetParsableVersionPart(string fullVersionName) => new([.. fullVersionName.TakeWhile(c => char.IsDigit(c) || c == '.')]); - private static CoreRuntime GetPlatformSpecific(CoreRuntime fallback, Assembly? assembly) - => TryGetTargetPlatform(assembly ?? Assembly.GetEntryAssembly(), out var platform) - ? new CoreRuntime(fallback.RuntimeMoniker, $"{fallback.MsBuildMoniker}-{platform}", fallback.Name) - : fallback; + private static CoreRuntime GetPlatformSpecific(Version version, Assembly? assembly) + => TryGetTargetPlatform(assembly, out var platform) + ? From(version, platform) + : version.Major switch + { + 5 => Core50, + 6 => Core60, + 7 => Core70, + 8 => Core80, + 9 => Core90, + 10 => Core10_0, + 11 => Core11_0, + _ => new(version), + }; private static bool TryGetTargetPlatform(Assembly? assembly, [NotNullWhen(true)] out string? platform) { @@ -253,7 +338,25 @@ private static bool TryGetTargetPlatform(Assembly? assembly, [NotNullWhen(true)] return false; platform = platformNameProperty.GetValue(attributeInstance) as string; - return platform.IsNotBlank(); + + // Read off the entry assembly, so it is not ours to trust. A malformed value is treated as no platform + // rather than left to throw out of From(): this runs inside a static initializer, where it would surface + // as a TypeInitializationException. + if (platform.IsBlank() || !IsValidPlatform(platform!)) + { + platform = null; + return false; + } + + return true; + } + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + { + if (benchmarkCase.Descriptor.Type.Assembly.IsLinqPad()) + return InProcessEmitToolchain.Default; + + return CsProjCoreToolchain.From(this, NetCoreAppSettings.Default); } } } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CoreWasmRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CoreWasmRuntime.cs new file mode 100644 index 0000000000..76bc5b4617 --- /dev/null +++ b/src/BenchmarkDotNet/Environments/Runtimes/CoreWasmRuntime.cs @@ -0,0 +1,24 @@ +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Wasm; +using System.ComponentModel; + +namespace BenchmarkDotNet.Environments; + +/// +/// The CoreCLR WebAssembly runtime. +/// +[EditorBrowsable(EditorBrowsableState.Never)] // WebAssembly with CoreCLR is still experimental. This is only used by dotnet contributors and dotnet/performance until official support is released. +public sealed class CoreWasmRuntime : WasmRuntime +{ + private CoreWasmRuntime(Version version) : base(version) { } + + public override string Name => "CoreWasm"; + + /// Returns a runtime for the given version. + // No cached versions yet (still experimental), so this always allocates. + public static CoreWasmRuntime From(Version version) => new(version); + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => CsProjCoreWasmToolchain.From(this, WasmSettings.Default); +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/CustomRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/CustomRuntime.cs deleted file mode 100644 index 96144b942a..0000000000 --- a/src/BenchmarkDotNet/Environments/Runtimes/CustomRuntime.cs +++ /dev/null @@ -1,12 +0,0 @@ -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Environments -{ - public abstract class CustomRuntime : Runtime - { - protected CustomRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) - : base(runtimeMoniker, msBuildMoniker, displayName) - { - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/LegacyMonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/LegacyMonoRuntime.cs new file mode 100644 index 0000000000..89b224be46 --- /dev/null +++ b/src/BenchmarkDotNet/Environments/Runtimes/LegacyMonoRuntime.cs @@ -0,0 +1,9 @@ +namespace BenchmarkDotNet.Environments; + +/// +/// A legacy (based on .Net Framework) Mono runtime. +/// +public abstract class LegacyMonoRuntime : Runtime +{ + public sealed override Version? Version => null; +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs deleted file mode 100644 index fd39fc9cd5..0000000000 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotLLVMRuntime.cs +++ /dev/null @@ -1,47 +0,0 @@ -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using System.ComponentModel; - -namespace BenchmarkDotNet.Environments -{ - public class MonoAotLLVMRuntime : Runtime, IEquatable - { - [EditorBrowsable(EditorBrowsableState.Never)] - internal static readonly MonoAotLLVMRuntime Default = new MonoAotLLVMRuntime(); - - public FileInfo AOTCompilerPath { get; } - public MonoAotCompilerMode AOTCompilerMode { get; } - - public override bool IsAOT => true; - - /// - /// creates new instance of MonoAotLLVMRuntime - /// - public MonoAotLLVMRuntime(FileInfo? aotCompilerPath, MonoAotCompilerMode aotCompilerMode, string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM", RuntimeMoniker moniker = RuntimeMoniker.MonoAOTLLVM) : base(moniker, msBuildMoniker, displayName) - { - ArgumentNullException.ThrowIfNull(aotCompilerPath); - - if (aotCompilerPath.IsNotNullButDoesNotExist()) - throw new FileNotFoundException($"Provided {nameof(aotCompilerPath)} file: \"{aotCompilerPath.FullName}\" doest NOT exist"); - - AOTCompilerPath = aotCompilerPath; - AOTCompilerMode = aotCompilerMode; - } - - // this ctor exists only for the purpose of having .Default property that returns something consumable by RuntimeInformation.GetCurrentRuntime() - private MonoAotLLVMRuntime(string msBuildMoniker = "net6.0", string displayName = "MonoAOTLLVM") : base(RuntimeMoniker.MonoAOTLLVM, msBuildMoniker, displayName) - { - AOTCompilerPath = new FileInfo("fake"); - } - - public override bool Equals(object? obj) - => obj is MonoAotLLVMRuntime other && Equals(other); - - public bool Equals(MonoAotLLVMRuntime? other) - => other != null && base.Equals(other) && other.AOTCompilerPath == AOTCompilerPath; - - public override int GetHashCode() - => HashCode.Combine(base.GetHashCode(), AOTCompilerPath); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoAotRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotRuntime.cs new file mode 100644 index 0000000000..8150b80658 --- /dev/null +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoAotRuntime.cs @@ -0,0 +1,20 @@ +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Mono; + +namespace BenchmarkDotNet.Environments; + +/// +/// The legacy (based on .Net Framework) Mono runtime, AOT-compiled via mono --aot. +/// +public sealed class MonoAotRuntime : LegacyMonoRuntime +{ + public static readonly MonoAotRuntime Default = new(); + + public override string Name => "MonoAot"; + + private MonoAotRuntime() { } + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => RoslynMonoAotToolchain.Default; +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoCoreRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoCoreRuntime.cs new file mode 100644 index 0000000000..634ec95a9d --- /dev/null +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoCoreRuntime.cs @@ -0,0 +1,52 @@ +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Mono; +using BenchmarkDotNet.Toolchains.NetCoreApp; + +namespace BenchmarkDotNet.Environments; + +/// +/// .NET running on the Mono VM (built with UseMonoRuntime=true). +/// +public sealed class MonoCoreRuntime : Runtime +{ + public static readonly MonoCoreRuntime Net60 = new(new(6, 0)); + public static readonly MonoCoreRuntime Net70 = new(new(7, 0)); + public static readonly MonoCoreRuntime Net80 = new(new(8, 0)); + public static readonly MonoCoreRuntime Net90 = new(new(9, 0)); + public static readonly MonoCoreRuntime Net10_0 = new(new(10, 0)); + public static readonly MonoCoreRuntime Net11_0 = new(new(11, 0)); + + private MonoCoreRuntime(Version version) => Version = ToRuntimeVersion(version); + + public override string Name => "Mono with .NET"; + + public override Version Version { get; } + + internal static MonoCoreRuntime GetCurrentVersion() => From(Environment.Version); + + /// Returns a runtime for the given version. + public static MonoCoreRuntime From(Version version) + => version.Major switch + { + 6 => Net60, + 7 => Net70, + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new(version), + }; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + { + // A .NET SDK with Mono as the default VM. Publishing self-contained apps might not work + // (https://github.com/dotnet/performance/issues/2787), so when the host is new Mono we use the default .NET + // toolchain, which performs a plain dotnet build that internally produces a Mono-based app. + if (RuntimeInformation.IsNewMono) + return CsProjCoreToolchain.From(From(Version)); + + return CsProjMonoCoreToolchain.From(this, MonoCoreSettings.Default); + } +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs index 137cc0ed37..ef919e7a2f 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoRuntime.cs @@ -1,73 +1,21 @@ -using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Mono; namespace BenchmarkDotNet.Environments { - public class MonoRuntime : Runtime, IEquatable + /// + /// The legacy (based on .Net Framework) Mono runtime. + /// + public sealed class MonoRuntime : LegacyMonoRuntime { - public static readonly MonoRuntime Default = new("Mono"); - public static readonly MonoRuntime Mono60 = new("Mono with .NET 6.0", RuntimeMoniker.Mono60, "net6.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono70 = new("Mono with .NET 7.0", RuntimeMoniker.Mono70, "net7.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono80 = new("Mono with .NET 8.0", RuntimeMoniker.Mono80, "net8.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono90 = new("Mono with .NET 9.0", RuntimeMoniker.Mono90, "net9.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono10_0 = new("Mono with .NET 10.0", RuntimeMoniker.Mono10_0, "net10.0", isDotNetBuiltIn: true); - public static readonly MonoRuntime Mono11_0 = new("Mono with .NET 11.0", RuntimeMoniker.Mono11_0, "net11.0", isDotNetBuiltIn: true); + public static readonly MonoRuntime Default = new(); - public string CustomPath { get; } = ""; + public override string Name => "Mono"; - public string AotArgs { get; } = ""; + private MonoRuntime() { } - public override bool IsAOT => !string.IsNullOrEmpty(AotArgs); - - public string MonoBclPath { get; } = ""; - - internal bool IsDotNetBuiltIn { get; } - - private MonoRuntime(string name) : base(RuntimeMoniker.Mono, "mono", name) { } - - private MonoRuntime(string name, RuntimeMoniker runtimeMoniker, string msBuildMoniker, bool isDotNetBuiltIn) : base(runtimeMoniker, msBuildMoniker, name) - { - IsDotNetBuiltIn = isDotNetBuiltIn; - } - - public MonoRuntime(string name, string customPath) : this(name) => CustomPath = customPath; - - public MonoRuntime(string name, string customPath, string aotArgs, string monoBclPath) : this(name) - { - CustomPath = customPath; - AotArgs = aotArgs; - MonoBclPath = monoBclPath; - } - - public override bool Equals(object? obj) - => obj is MonoRuntime runtime && Equals(runtime); - - public bool Equals(MonoRuntime? other) - { - if (other is null) - return false; - if (ReferenceEquals(this, other)) - return true; - - return base.Equals(other) - && Name == other.Name - && CustomPath == other.CustomPath - && AotArgs == other.AotArgs - && MonoBclPath == other.MonoBclPath; - } - - public override int GetHashCode() - => HashCode.Combine(base.GetHashCode(), Name, CustomPath, AotArgs, MonoBclPath); - - internal static Runtime GetCurrentVersion() - { - Version version = Environment.Version; - return version.Major switch - { - 6 => Mono60, - 7 => Mono70, - 8 => Mono80, - _ => new MonoRuntime($"Mono with .NET {version.Major}.{version.Minor}", RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", isDotNetBuiltIn: true) - }; - } + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => RoslynMonoToolchain.Default; } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoWasmAotRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoWasmAotRuntime.cs new file mode 100644 index 0000000000..154d9244f0 --- /dev/null +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoWasmAotRuntime.cs @@ -0,0 +1,34 @@ +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Wasm; + +namespace BenchmarkDotNet.Environments; + +/// +/// The Mono WebAssembly runtime AOT-compiled. +/// +public sealed class MonoWasmAotRuntime : WasmRuntime +{ + public static readonly MonoWasmAotRuntime Net80 = new(new(8, 0)); + public static readonly MonoWasmAotRuntime Net90 = new(new(9, 0)); + public static readonly MonoWasmAotRuntime Net10_0 = new(new(10, 0)); + public static readonly MonoWasmAotRuntime Net11_0 = new(new(11, 0)); + + private MonoWasmAotRuntime(Version version) : base(version) { } + + public override string Name => "MonoWasmAot"; + + /// Returns a runtime for the given version. + public static MonoWasmAotRuntime From(Version version) + => version.Major switch + { + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new(version), + }; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => CsProjMonoWasmAotToolchain.From(this, WasmSettings.Default); +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/MonoWasmRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/MonoWasmRuntime.cs new file mode 100644 index 0000000000..4e7f6d05d4 --- /dev/null +++ b/src/BenchmarkDotNet/Environments/Runtimes/MonoWasmRuntime.cs @@ -0,0 +1,34 @@ +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Wasm; + +namespace BenchmarkDotNet.Environments; + +/// +/// The Mono WebAssembly runtime running in interpreter mode (including the Jiterpreter). +/// +public sealed class MonoWasmRuntime : WasmRuntime +{ + public static readonly MonoWasmRuntime Net80 = new(new(8, 0)); + public static readonly MonoWasmRuntime Net90 = new(new(9, 0)); + public static readonly MonoWasmRuntime Net10_0 = new(new(10, 0)); + public static readonly MonoWasmRuntime Net11_0 = new(new(11, 0)); + + private MonoWasmRuntime(Version version) : base(version) { } + + public override string Name => "MonoWasm"; + + /// Returns a runtime for the given version. + public static MonoWasmRuntime From(Version version) + => version.Major switch + { + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new(version), + }; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => CsProjMonoWasmToolchain.From(this, WasmSettings.Default); +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs index 56634c752a..23badab98f 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/NativeAotRuntime.cs @@ -1,60 +1,62 @@ -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.NativeAot; namespace BenchmarkDotNet.Environments { - public class NativeAotRuntime : Runtime + public sealed class NativeAotRuntime : Runtime { /// /// NativeAOT compiled as net7.0 /// - public static readonly NativeAotRuntime Net70 = new NativeAotRuntime(RuntimeMoniker.NativeAot70, "net7.0", "NativeAOT 7.0"); + public static readonly NativeAotRuntime Net70 = new(new(7, 0)); /// /// NativeAOT compiled as net8.0 /// - public static readonly NativeAotRuntime Net80 = new NativeAotRuntime(RuntimeMoniker.NativeAot80, "net8.0", "NativeAOT 8.0"); + public static readonly NativeAotRuntime Net80 = new(new(8, 0)); /// /// NativeAOT compiled as net9.0 /// - public static readonly NativeAotRuntime Net90 = new NativeAotRuntime(RuntimeMoniker.NativeAot90, "net9.0", "NativeAOT 9.0"); + public static readonly NativeAotRuntime Net90 = new(new(9, 0)); /// /// NativeAOT compiled as net10.0 /// - public static readonly NativeAotRuntime Net10_0 = new NativeAotRuntime(RuntimeMoniker.NativeAot10_0, "net10.0", "NativeAOT 10.0"); + public static readonly NativeAotRuntime Net10_0 = new(new(10, 0)); /// /// NativeAOT compiled as net11.0 /// - public static readonly NativeAotRuntime Net11_0 = new NativeAotRuntime(RuntimeMoniker.NativeAot11_0, "net11.0", "NativeAOT 11.0"); + public static readonly NativeAotRuntime Net11_0 = new(new(11, 0)); - public override bool IsAOT => true; + private NativeAotRuntime(Version version) => Version = ToRuntimeVersion(version); - private NativeAotRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) - : base(runtimeMoniker, msBuildMoniker, displayName) - { - } + public override string Name => "NativeAOT"; + + public override Version Version { get; } - public static NativeAotRuntime GetCurrentVersion() + internal static NativeAotRuntime GetCurrentVersion() { if (!RuntimeInformation.IsNetCore && !RuntimeInformation.IsNativeAOT) { throw new NotSupportedException("It's impossible to reliably detect the version of NativeAOT if the process is not a .NET or NativeAOT process!"); } - if (!CoreRuntime.TryGetVersion(out var version)) - { - throw new NotSupportedException("Failed to recognize NativeAOT version"); - } + return From(Environment.Version); + } - switch (version) + /// Returns a runtime for the given version. + public static NativeAotRuntime From(Version version) + => version.Major switch { - case Version v when v.Major == 7 && v.Minor == 0: return Net70; - case Version v when v.Major == 8 && v.Minor == 0: return Net80; - case Version v when v.Major == 9 && v.Minor == 0: return Net90; - case Version v when v.Major == 10 && v.Minor == 0: return Net10_0; - case Version v when v.Major == 11 && v.Minor == 0: return Net11_0; - default: - return new NativeAotRuntime(RuntimeMoniker.NotRecognized, $"net{version.Major}.{version.Minor}", $"NativeAOT {version.Major}.{version.Minor}"); - } - } + 7 => Net70, + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new(version), + }; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => CsProjNativeAotToolchain.From(this, NativeAotSettings.Default); } } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/R2RRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/R2RRuntime.cs index 450a6f85f7..8c4a81f41a 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/R2RRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/R2RRuntime.cs @@ -1,17 +1,34 @@ -using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.R2R; namespace BenchmarkDotNet.Environments { - public class R2RRuntime : Runtime + public sealed class R2RRuntime : Runtime { - public static readonly R2RRuntime Net80 = new R2RRuntime(RuntimeMoniker.R2R80, "net8.0", "R2R 8.0"); - public static readonly R2RRuntime Net90 = new R2RRuntime(RuntimeMoniker.R2R90, "net9.0", "R2R 9.0"); - public static readonly R2RRuntime Net10_0 = new R2RRuntime(RuntimeMoniker.R2R10_0, "net10.0", "R2R 10.0"); - public static readonly R2RRuntime Net11_0 = new R2RRuntime(RuntimeMoniker.R2R11_0, "net11.0", "R2R 11.0"); + public static readonly R2RRuntime Net80 = new(new(8, 0)); + public static readonly R2RRuntime Net90 = new(new(9, 0)); + public static readonly R2RRuntime Net10_0 = new(new(10, 0)); + public static readonly R2RRuntime Net11_0 = new(new(11, 0)); - private R2RRuntime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) - : base(runtimeMoniker, msBuildMoniker, displayName) - { - } + private R2RRuntime(Version version) => Version = ToRuntimeVersion(version); + + public override string Name => "R2R"; + + public override Version Version { get; } + + /// Returns a runtime for the given version. + public static R2RRuntime From(Version version) + => version.Major switch + { + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new(version), + }; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => CsProjR2RToolchain.From(this, R2RSettings.Default); } } diff --git a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs index 822ff99bba..2cc28a6b7f 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/Runtime.cs @@ -1,45 +1,183 @@ -using BenchmarkDotNet.Jobs; -using JetBrains.Annotations; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; +using System.Diagnostics.CodeAnalysis; namespace BenchmarkDotNet.Environments { - public abstract class Runtime : IEquatable + /// + /// Describes a .NET runtime (e.g. .NET Core, .NET Framework, Mono, NativeAOT) that benchmarks can target. + /// + public abstract class Runtime { /// - /// Display name + /// The name of the runtime. /// - [PublicAPI] - public string Name { get; } + public abstract string Name { get; } /// - /// Target Framework Moniker + /// The version of the runtime, or if it is unknown. /// - public RuntimeMoniker RuntimeMoniker { get; } + public abstract Version? Version { get; } /// - /// MsBuild Target Framework Moniker, example: net462, net8.0 + /// Drops the components past the ones that identify a release, which are servicing detail - Environment.Version + /// reports 8.0.30 - and would otherwise reach the Runtime column, job ids and artifact file names and change + /// with every OS patch. Also what makes 8.0, 8.0.0 and 8.0.0.0 one runtime rather than three. /// - public string MsBuildMoniker { get; } + /// The version to normalize. + /// + /// For .NET Framework, whose releases are identified by Major.Minor.Build (4.8.1 is not 4.8), leaving only the + /// revision as servicing detail. + /// + private protected static Version ToRuntimeVersion(Version version, bool keepBuild = false) + => keepBuild && version.Build > 0 + ? new(version.Major, version.Minor, version.Build) + : new(version.Major, version.Minor); - public virtual bool IsAOT => false; + /// + /// Determines whether the specified object is a of the same type with equal and . + /// Concrete runtimes override this to also compare any additional state; this is the single equality entry point, + /// so all comparison paths (including ) go through it. + /// + public override bool Equals(object? obj) + => obj is Runtime other + && other.GetType() == GetType() + && Name == other.Name + && Version == other.Version; + + /// + /// Returns a hash code derived from the runtime type, and . + /// + public override int GetHashCode() + => HashCode.Combine(GetType(), Name, Version); + + /// + /// Returns the , followed by the when it is known. + /// + public override string ToString() + => Version == null ? Name : $"{Name} {Version}"; + + /// + /// Returns the default used to build and run benchmarks for this runtime when the job + /// doesn't specify one explicitly. Custom runtimes should override this to provide their own toolchain. + /// + /// + /// The benchmark the toolchain is being resolved for; runtimes may inspect its job characteristics or the + /// descriptor's target assembly. + /// + public abstract IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase); + + /// + /// Parses a runtime moniker string (e.g. net8.0, net472, netcoreapp3.1, nativeaot8.0, + /// mono8.0, monowasm8.0, r2r8.0) into the corresponding . Both the dotted + /// TFM spelling and the compact spelling (e.g. net80) are accepted, case-insensitively. + /// + /// The moniker does not map to a known runtime. + public static Runtime Parse(string moniker) + => TryParse(moniker, out var runtime) + ? runtime + : throw new ArgumentException($"Unable to parse '{moniker}' into a known runtime.", nameof(moniker)); - protected Runtime(RuntimeMoniker runtimeMoniker, string msBuildMoniker, string displayName) + /// + /// Attempts to parse a runtime moniker string into the corresponding . See . + /// + public static bool TryParse(string moniker, [NotNullWhen(true)] out Runtime? runtime) { - if (string.IsNullOrEmpty(displayName)) throw new ArgumentNullException(nameof(displayName)); - if (string.IsNullOrEmpty(msBuildMoniker)) throw new ArgumentNullException(nameof(msBuildMoniker)); + runtime = null; + if (moniker.IsBlank()) + return false; - RuntimeMoniker = runtimeMoniker; - MsBuildMoniker = msBuildMoniker; - Name = displayName; - } + string s = moniker.Trim(); + + // Split off a target-platform suffix (e.g. "net8.0-windows"); only net5.0+ has them. Everything + // CoreRuntime.From would throw on is reported as "not a runtime" instead, here and below. + string? platform = null; + int dash = s.IndexOf('-'); + if (dash >= 0) + { + platform = s[(dash + 1)..]; + s = s[..dash]; - public override string ToString() => Name; + if (!CoreRuntime.IsValidPlatform(platform)) + return false; + } - public bool Equals(Runtime? other) - => other != null && other.Name == Name && other.MsBuildMoniker == MsBuildMoniker && other.RuntimeMoniker == RuntimeMoniker; + // Prefixes, most-specific first (so "monowasmaot"/"monoaot" beat "monowasm"/bare "mono", and "netcoreapp" + // beats "net"). Both the moniker VALUES ("monowasm8.0") and the RuntimeMoniker field-name spellings + // ("MonoWasm80") parse, since TryParseVersion accepts dotted, compact, and '_'-separated versions. + if (TryStripPrefix(s, "nativeaot", out string rest) && TryParseVersion(rest, out var version)) + runtime = NativeAotRuntime.From(version); + else if (TryStripPrefix(s, "monowasmaot", out rest) && TryParseVersion(rest, out version)) + runtime = MonoWasmAotRuntime.From(version); + else if (TryStripPrefix(s, "monowasm", out rest) && TryParseVersion(rest, out version)) + runtime = MonoWasmRuntime.From(version); + else if (TryStripPrefix(s, "corewasm", out rest) && TryParseVersion(rest, out version)) + runtime = CoreWasmRuntime.From(version); + else if (TryStripPrefix(s, "monoaot", out rest) && rest.Length == 0) + // Legacy Mono AOT is versionless (like classic Mono); the new Mono AOT ("monoaotX.Y") has no public toolchain. + runtime = MonoAotRuntime.Default; + else if (TryStripPrefix(s, "mono", out rest)) + // Bare "mono" is the classic Mono VM; "monoX.Y" is .NET on the Mono VM. Anything else (e.g. "monovm") isn't Mono. + runtime = rest.Length == 0 ? MonoRuntime.Default + : TryParseVersion(rest, out version) ? MonoCoreRuntime.From(version) + : null; + else if (TryStripPrefix(s, "r2r", out rest) && TryParseVersion(rest, out version)) + runtime = R2RRuntime.From(version); + else if (TryStripPrefix(s, "netcoreapp", out rest) && TryParseVersion(rest, out version)) + runtime = CoreRuntime.From(version); + else if (TryStripPrefix(s, "net", out rest) && TryParseVersion(rest, out version)) + runtime = version.Major == 4 + ? ClrRuntime.FromVersion(version) + : CoreRuntime.From(version, version.Major >= 5 ? platform : null); - public override bool Equals(object? obj) => obj is Runtime other && Equals(other); + // Every branch above except net5.0+ ignores the platform. Silently dropping it would benchmark something + // other than what was asked for, so a suffix that did not take effect rejects the moniker. + if (platform != null && (runtime as CoreRuntime)?.IsPlatformSpecific != true) + runtime = null; - public override int GetHashCode() => HashCode.Combine(Name, MsBuildMoniker, RuntimeMoniker); + return runtime != null; + + static bool TryStripPrefix(string value, string prefix, out string rest) + { + if (value.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + { + rest = value[prefix.Length..]; + return true; + } + rest = string.Empty; + return false; + } + + static bool TryParseVersion(string value, [NotNullWhen(true)] out Version? version) + { + version = null!; + if (value.IsBlank()) + return false; + + value = value.Replace('_', '.'); + if (value.Contains('.')) + return Version.TryParse(value, out version); + + foreach (char c in value) + if (!char.IsDigit(c)) + return false; + + // Compact spellings: "80" -> 8.0, "472" -> 4.7.2, "20" -> 2.0. + version = value.Length switch + { + 1 => new Version(value[0] - '0', 0), + 2 => new Version(value[0] - '0', value[1] - '0'), + 3 => new Version(value[0] - '0', value[1] - '0', value[2] - '0'), + _ => null, + }; + // Compact spellings only express single-digit majors; .NET 10+ has no unambiguous compact form + // ("net10" would parse as 1.0), and no BDN-supported runtime has a major below 2, so reject those + // and require a separator instead (e.g. "net10.0" or "Net10_0"). + if (version != null && version.Major < 2) + version = null; + return version != null; + } + } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/UnknownRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/UnknownRuntime.cs new file mode 100644 index 0000000000..4fc8b8b992 --- /dev/null +++ b/src/BenchmarkDotNet/Environments/Runtimes/UnknownRuntime.cs @@ -0,0 +1,18 @@ +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Environments; + +public sealed class UnknownRuntime : Runtime +{ + public static readonly UnknownRuntime Instance = new(); + + private UnknownRuntime() { } + + public override string Name => "?"; + + public override Version? Version => null; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) + => throw new NotSupportedException($"A default toolchain cannot be determined for {nameof(UnknownRuntime)}."); +} diff --git a/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs b/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs index f0604fdbea..82bdb65922 100644 --- a/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs +++ b/src/BenchmarkDotNet/Environments/Runtimes/WasmRuntime.cs @@ -1,123 +1,19 @@ -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.MonoWasm; -using System.ComponentModel; namespace BenchmarkDotNet.Environments { - public class WasmRuntime : Runtime, IEquatable + public abstract class WasmRuntime(Version version) : Runtime { - public delegate string ArgumentFormatter(WasmRuntime runtime, ArtifactsPaths artifactsPaths, string args); - - [EditorBrowsable(EditorBrowsableState.Never)] - internal static readonly WasmRuntime Default = new WasmRuntime(); - - public string JavaScriptEngine { get; } - - public string JavaScriptEngineArguments { get; } - - public ArgumentFormatter JavaScriptEngineArgumentFormatter { get; } - - public override bool IsAOT { get; } - - /// - /// Specifies the runtime flavor used for WASM benchmarks. (default) resolves the - /// Mono runtime pack (Microsoft.NETCore.App.Runtime.Mono.browser-wasm); resolves - /// the CoreCLR runtime pack (Microsoft.NETCore.App.Runtime.browser-wasm). - /// - public RuntimeFlavor RuntimeFlavor { get; } - - /// - /// Maximum time in minutes to wait for a single benchmark process to finish before force killing it. Default is 10 minutes. - /// - public int ProcessTimeoutMinutes { get; } - - /// - /// Specifies the IPC mechanism to use. Default is which automatically detects the JavaScript engine capabilities. - /// - public WasmIpcType IpcType { get; } - - public FileInfo? MainJsTemplate { get; set; } - - /// - /// creates new instance of WasmRuntime - /// - /// moniker - /// Runtime moniker - /// display name - /// Specifies whether AOT or Interpreter project should be generated. - /// Full path to a java script engine used to run the benchmarks. - /// Arguments for the javascript engine. - /// Runtime flavor to use: Mono (default) or CoreCLR. - /// Maximum time in minutes to wait for a single benchmark process to finish. Default is 10. - /// Optional custom template for the generated main.mjs file. If not provided, a default template will be used. - /// IPC mechanism to use. Default is Auto which detects based on JavaScript engine. - /// Allows to format or customize the arguments passed to the javascript engine. - public WasmRuntime( - string msBuildMoniker, - RuntimeMoniker moniker, - string displayName, - bool aot, - string javaScriptEngine, - string? javaScriptEngineArguments = "", - RuntimeFlavor runtimeFlavor = RuntimeFlavor.Mono, - int processTimeoutMinutes = 10, - FileInfo? mainJsTemplate = null, - WasmIpcType ipcType = WasmIpcType.Auto, - ArgumentFormatter? javaScriptEngineArgumentFormatter = null) : base(moniker, msBuildMoniker, displayName) - { - // Resolve path for windows because we can't use ProcessStartInfo.UseShellExecute while redirecting std out in the executor. - if (!ProcessHelper.TryResolveExecutableInPath(javaScriptEngine, out javaScriptEngine!)) - throw new FileNotFoundException($"Provided {nameof(javaScriptEngine)} file: \"{javaScriptEngine}\" does NOT exist"); - - JavaScriptEngine = javaScriptEngine; - JavaScriptEngineArguments = javaScriptEngineArguments ?? ""; - JavaScriptEngineArgumentFormatter = javaScriptEngineArgumentFormatter ?? DefaultArgumentFormatter; - RuntimeFlavor = runtimeFlavor; - IsAOT = aot; - ProcessTimeoutMinutes = processTimeoutMinutes; - IpcType = ipcType; - MainJsTemplate = mainJsTemplate; - } - - private WasmRuntime() : base(RuntimeMoniker.WasmNet80, "Wasm", "Wasm") - { - IsAOT = RuntimeInformation.IsAot; - JavaScriptEngine = ""; - JavaScriptEngineArguments = ""; - ProcessTimeoutMinutes = 10; - IpcType = WasmIpcType.Auto; - JavaScriptEngineArgumentFormatter = DefaultArgumentFormatter; - } - - public override bool Equals(object? obj) - => obj is WasmRuntime other && Equals(other); - - public bool Equals(WasmRuntime? other) - { - return other != null - && base.Equals(other) - && other.JavaScriptEngine == JavaScriptEngine - && other.JavaScriptEngineArguments == JavaScriptEngineArguments - && other.JavaScriptEngineArgumentFormatter == JavaScriptEngineArgumentFormatter - && other.IsAOT == IsAOT - && other.ProcessTimeoutMinutes == ProcessTimeoutMinutes - && other.RuntimeFlavor == RuntimeFlavor - && other.IpcType == IpcType; - } - - public override int GetHashCode() - => HashCode.Combine(base.GetHashCode(), JavaScriptEngine, JavaScriptEngineArguments, JavaScriptEngineArgumentFormatter, IsAOT, RuntimeFlavor, ProcessTimeoutMinutes, IpcType); - - private static string DefaultArgumentFormatter(WasmRuntime runtime, ArtifactsPaths artifactsPaths, string args) - { - return Path.GetFileNameWithoutExtension(runtime.JavaScriptEngine).ToLower() switch - { - "node" or "bun" => $"{runtime.JavaScriptEngineArguments} {artifactsPaths.ExecutablePath} -- --run {artifactsPaths.ProgramName}.dll {args}", - _ => $"{runtime.JavaScriptEngineArguments} --module {artifactsPaths.ExecutablePath} -- --run {artifactsPaths.ProgramName}.dll {args}", - }; - } + public override Version Version { get; } = ToRuntimeVersion(version); + + // Resolves the concrete WebAssembly runtime for the current process. CoreCLR vs Mono is detected reliably via + // RuntimeInformation.IsMono; the version comes from Environment.Version like the other Mono-based runtimes. + // Interpreter vs AOT can NOT be detected at runtime - AOT is a publish-time setting and AOT'd wasm still bundles + // the interpreter, so IsAot/RuntimeFeature report the same for both - so Mono wasm defaults to the interpreter + // runtime here. When AOT is used the job's toolchain (CsProjMonoWasmAotToolchain) supplies the authoritative runtime. + internal static WasmRuntime GetCurrentVersion() + => RuntimeInformation.IsMono + ? MonoWasmRuntime.From(Environment.Version) + : CoreWasmRuntime.From(Environment.Version); } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs index 6f104ee23d..0460afb936 100644 --- a/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs +++ b/src/BenchmarkDotNet/Extensions/ProcessExtensions.cs @@ -1,7 +1,6 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; @@ -108,12 +107,6 @@ public static bool TrySetAffinity( internal static void SetEnvironmentVariables(this ProcessStartInfo start, BenchmarkCase benchmarkCase, IResolver resolver) { - if (benchmarkCase.Job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.Version.IsNotBlank()) - SetClrEnvironmentVariables(start, "Version", clrRuntime.Version); - - if (benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime && monoRuntime.MonoBclPath.IsNotBlank()) - start.Environment["MONO_PATH"] = monoRuntime.MonoBclPath; - if (benchmarkCase.Config.HasPerfCollectProfiler()) { // enable tracing configuration inside of CoreCLR (https://github.com/dotnet/coreclr/blob/master/Documentation/project-docs/linux-performance-tracing.md#collecting-a-trace) @@ -128,7 +121,7 @@ internal static void SetEnvironmentVariables(this ProcessStartInfo start, Benchm // corerun does not understand runtimeconfig.json files; // we have to set "COMPlus_GC*" environment variables as documented in // https://docs.microsoft.com/en-us/dotnet/core/run-time-config/garbage-collector - if (benchmarkCase.Job.Infrastructure.Toolchain is CoreRunToolchain _) + if (benchmarkCase.Job.Infrastructure.Toolchain is CoreRunToolchain) start.SetCoreRunEnvironmentVariables(benchmarkCase, resolver); // disable ReSharper's Dynamic Program Analysis (see https://github.com/dotnet/BenchmarkDotNet/issues/1871 for details) @@ -268,7 +261,7 @@ private static void SetCoreRunEnvironmentVariables(this ProcessStartInfo start, SetClrEnvironmentVariables(start, "GCHeapCount", gcMode.HeapCount.ToString("X")); } - private static void SetClrEnvironmentVariables(ProcessStartInfo start, string suffix, string value) + internal static void SetClrEnvironmentVariables(this ProcessStartInfo start, string suffix, string value) { start.Environment[$"DOTNET_{suffix}"] = value; start.Environment[$"COMPlus_{suffix}"] = value; diff --git a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs b/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs deleted file mode 100644 index c483b9749b..0000000000 --- a/src/BenchmarkDotNet/Extensions/RuntimeMonikerExtensions.cs +++ /dev/null @@ -1,137 +0,0 @@ -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; - -namespace BenchmarkDotNet.Extensions -{ - internal static class RuntimeMonikerExtensions - { - internal static Runtime GetRuntime(this RuntimeMoniker runtimeMoniker) - { - switch (runtimeMoniker) - { - case RuntimeMoniker.Net461: - return ClrRuntime.Net461; - case RuntimeMoniker.Net462: - return ClrRuntime.Net462; - case RuntimeMoniker.Net47: - return ClrRuntime.Net47; - case RuntimeMoniker.Net471: - return ClrRuntime.Net471; - case RuntimeMoniker.Net472: - return ClrRuntime.Net472; - case RuntimeMoniker.Net48: - return ClrRuntime.Net48; - case RuntimeMoniker.Net481: - return ClrRuntime.Net481; - case RuntimeMoniker.NetCoreApp20: - return CoreRuntime.Core20; - case RuntimeMoniker.NetCoreApp21: - return CoreRuntime.Core21; - case RuntimeMoniker.NetCoreApp22: - return CoreRuntime.Core22; - case RuntimeMoniker.NetCoreApp30: - return CoreRuntime.Core30; - case RuntimeMoniker.NetCoreApp31: - return CoreRuntime.Core31; - case RuntimeMoniker.Net50: - return CoreRuntime.Core50; - case RuntimeMoniker.Net60: - return CoreRuntime.Core60; - case RuntimeMoniker.Net70: - return CoreRuntime.Core70; - case RuntimeMoniker.Net80: - return CoreRuntime.Core80; - case RuntimeMoniker.Net90: - return CoreRuntime.Core90; - case RuntimeMoniker.Net10_0: - return CoreRuntime.Core10_0; - case RuntimeMoniker.Net11_0: - return CoreRuntime.Core11_0; - case RuntimeMoniker.Mono: - return MonoRuntime.Default; - case RuntimeMoniker.NativeAot70: - return NativeAotRuntime.Net70; - case RuntimeMoniker.NativeAot80: - return NativeAotRuntime.Net80; - case RuntimeMoniker.NativeAot90: - return NativeAotRuntime.Net90; - case RuntimeMoniker.NativeAot10_0: - return NativeAotRuntime.Net10_0; - case RuntimeMoniker.NativeAot11_0: - return NativeAotRuntime.Net11_0; - case RuntimeMoniker.Mono60: - return MonoRuntime.Mono60; - case RuntimeMoniker.Mono70: - return MonoRuntime.Mono70; - case RuntimeMoniker.Mono80: - return MonoRuntime.Mono80; - case RuntimeMoniker.Mono90: - return MonoRuntime.Mono90; - case RuntimeMoniker.Mono10_0: - return MonoRuntime.Mono10_0; - case RuntimeMoniker.Mono11_0: - return MonoRuntime.Mono11_0; - case RuntimeMoniker.R2R80: - return R2RRuntime.Net80; - case RuntimeMoniker.R2R90: - return R2RRuntime.Net90; - case RuntimeMoniker.R2R10_0: - return R2RRuntime.Net10_0; - case RuntimeMoniker.R2R11_0: - return R2RRuntime.Net11_0; - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "Runtime Moniker not supported"); - } - } - - internal static Version GetRuntimeVersion(this RuntimeMoniker runtimeMoniker) => runtimeMoniker switch - { - RuntimeMoniker.Net461 => new Version(4, 6, 1), - RuntimeMoniker.Net462 => new Version(4, 6, 2), - RuntimeMoniker.Net47 => new Version(4, 7), - RuntimeMoniker.Net471 => new Version(4, 7, 1), - RuntimeMoniker.Net472 => new Version(4, 7, 2), - RuntimeMoniker.Net48 => new Version(4, 8), - RuntimeMoniker.Net481 => new Version(4, 8, 1), - RuntimeMoniker.NetCoreApp20 => new Version(2, 0), - RuntimeMoniker.NetCoreApp21 => new Version(2, 1), - RuntimeMoniker.NetCoreApp22 => new Version(2, 2), - RuntimeMoniker.NetCoreApp30 => new Version(3, 0), - RuntimeMoniker.NetCoreApp31 => new Version(3, 1), - RuntimeMoniker.Net50 => new Version(5, 0), - RuntimeMoniker.Net60 => new Version(6, 0), - RuntimeMoniker.Net70 => new Version(7, 0), - RuntimeMoniker.Net80 => new Version(8, 0), - RuntimeMoniker.Net90 => new Version(9, 0), - RuntimeMoniker.Net10_0 => new Version(10, 0), - RuntimeMoniker.Net11_0 => new Version(11, 0), - RuntimeMoniker.NativeAot70 => new Version(7, 0), - RuntimeMoniker.NativeAot80 => new Version(8, 0), - RuntimeMoniker.NativeAot90 => new Version(9, 0), - RuntimeMoniker.NativeAot10_0 => new Version(10, 0), - RuntimeMoniker.NativeAot11_0 => new Version(11, 0), - RuntimeMoniker.Mono60 => new Version(6, 0), - RuntimeMoniker.Mono70 => new Version(7, 0), - RuntimeMoniker.Mono80 => new Version(8, 0), - RuntimeMoniker.Mono90 => new Version(9, 0), - RuntimeMoniker.Mono10_0 => new Version(10, 0), - RuntimeMoniker.Mono11_0 => new Version(11, 0), - RuntimeMoniker.WasmNet80 => new Version(8, 0), - RuntimeMoniker.WasmNet90 => new Version(9, 0), - RuntimeMoniker.WasmNet10_0 => new Version(10, 0), - RuntimeMoniker.WasmNet11_0 => new Version(11, 0), - RuntimeMoniker.MonoAOTLLVM => Portability.RuntimeInformation.IsNetCore && CoreRuntime.TryGetVersion(out var version) ? version : new Version(6, 0), - RuntimeMoniker.MonoAOTLLVMNet60 => new Version(6, 0), - RuntimeMoniker.MonoAOTLLVMNet70 => new Version(7, 0), - RuntimeMoniker.MonoAOTLLVMNet80 => new Version(8, 0), - RuntimeMoniker.MonoAOTLLVMNet90 => new Version(9, 0), - RuntimeMoniker.MonoAOTLLVMNet10_0 => new Version(10, 0), - RuntimeMoniker.MonoAOTLLVMNet11_0 => new Version(11, 0), - RuntimeMoniker.R2R80 => new Version(8, 0), - RuntimeMoniker.R2R90 => new Version(9, 0), - RuntimeMoniker.R2R10_0 => new Version(10, 0), - RuntimeMoniker.R2R11_0 => new Version(11, 0), - _ => throw new NotImplementedException($"{nameof(GetRuntimeVersion)} not implemented for {runtimeMoniker}") - }; - } -} diff --git a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs index 7885a08b39..8775c97d2f 100644 --- a/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs +++ b/src/BenchmarkDotNet/Helpers/ArtifactFileNameHelper.cs @@ -3,8 +3,8 @@ using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers.Hashing; +using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains; namespace BenchmarkDotNet.Helpers { @@ -69,9 +69,34 @@ private static string GetLimitedFilePath(DiagnoserActionParameters details, stri private static string GetFilePath(string fileName, DiagnoserActionParameters details, string? subfolder, DateTime? creationTime, string fileExtension) { - // if we run for more than one toolchain, the output file name should contain the name too so we can differ net462 vs net8.0 etc - if (details.Config.GetJobs().Select(job => ToolchainExtensions.GetToolchain(job)).Distinct().Count() > 1) - fileName += $"-{details.BenchmarkCase.Job.Environment.Runtime?.Name ?? details.BenchmarkCase.GetToolchain()?.Name ?? details.BenchmarkCase.Job.Id}"; + // Disambiguate output file names across the config's jobs (JobComparer has already made them distinct), + // so a benchmark that runs under more than one job doesn't produce colliding files. If the jobs differ + // in runtime, tag every file with its runtime; expand to the toolchain when jobs sharing a runtime + // configure different toolchains (jobs that don't configure one resolve to the same default for a given + // runtime); and finally fall back to the job index (always unique) when neither separates this job from + // another (e.g. jobs differing only by toolchain settings or some other characteristic). + var jobs = details.Config.GetJobs().ToArray(); + if (jobs.Length > 1) + { + bool runtimesDiffer = jobs.Select(job => job.GetRuntime()).Distinct().Count() > 1; + bool toolchainsDiffer = jobs + .GroupBy(job => job.GetRuntime()) + .Any(group => group.Select(job => job.Infrastructure.TryGetToolchain(out var toolchain) ? toolchain.ToString() : null).Distinct().Count() > 1); + + string Disambiguator(Job job) + { + string result = runtimesDiffer ? $"-{job.GetRuntime()}" : string.Empty; + if (toolchainsDiffer && job.Infrastructure.TryGetToolchain(out var toolchain)) + result += $"-{toolchain}"; + return result; + } + + string suffix = Disambiguator(details.BenchmarkCase.Job); + fileName += suffix; + + if (jobs.Count(job => Disambiguator(job) == suffix) > 1) + fileName += $"-{Array.IndexOf(jobs, details.BenchmarkCase.Job)}"; + } if (creationTime.HasValue) fileName += $"-{creationTime.Value.ToString(BenchmarkRunnerClean.DateTimeFormat)}"; diff --git a/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs index d0f249cc09..c107a4b1e9 100644 --- a/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs +++ b/src/BenchmarkDotNet/Helpers/DisposeAtProcessTermination.cs @@ -29,7 +29,7 @@ public abstract class DisposeAtProcessTermination : IDisposable { // Cancel key presses are not supported by .NET or do not exist for these platforms. internal static readonly bool ConsoleSupportsCancelKeyPress = - !(OsDetector.IsAndroid() || OsDetector.IsIOS() || OsDetector.IsTvOS() || Portability.RuntimeInformation.IsWasm); + !(OsDetector.IsMobile() || Portability.RuntimeInformation.IsWasm); private static int cancelKeyPressKeepAliveRegistered; diff --git a/src/BenchmarkDotNet/Helpers/FrameworkVersionHelper.cs b/src/BenchmarkDotNet/Helpers/FrameworkVersionHelper.cs index cd1c0e5525..59c41a17e7 100644 --- a/src/BenchmarkDotNet/Helpers/FrameworkVersionHelper.cs +++ b/src/BenchmarkDotNet/Helpers/FrameworkVersionHelper.cs @@ -8,28 +8,28 @@ internal static class FrameworkVersionHelper { // magic numbers come from https://docs.microsoft.com/en-us/dotnet/framework/migration-guide/how-to-determine-which-versions-are-installed // should be ordered by release number - private static readonly (int minReleaseNumber, string version)[] FrameworkVersions = + private static readonly (int minReleaseNumber, Version version)[] FrameworkVersions = [ - (533320, "4.8.1"), // value taken from Windows 11 arm64 insider build - (528040, "4.8"), - (461808, "4.7.2"), - (461308, "4.7.1"), - (460798, "4.7"), - (394802, "4.6.2"), - (394254, "4.6.1") + (533320, new(4,8,1)), + (528040, new(4,8)), + (461808, new(4,7,2)), + (461308, new(4,7,1)), + (460798, new(4,7)), + (394802, new(4,6,2)), + (394254, new(4,6,1)) ]; - internal static string? GetTargetFrameworkVersion(Assembly? assembly) + internal static Version? GetTargetFrameworkVersion(Assembly? assembly) // Look for a TargetFrameworkAttribute with a supported Framework version. => assembly?.GetCustomAttribute()?.FrameworkName switch { - ".NETFramework,Version=v4.6.1" => "4.6.1", - ".NETFramework,Version=v4.6.2" => "4.6.2", - ".NETFramework,Version=v4.7" => "4.7", - ".NETFramework,Version=v4.7.1" => "4.7.1", - ".NETFramework,Version=v4.7.2" => "4.7.2", - ".NETFramework,Version=v4.8" => "4.8", - ".NETFramework,Version=v4.8.1" => "4.8.1", + ".NETFramework,Version=v4.6.1" => new(4, 6, 1), + ".NETFramework,Version=v4.6.2" => new(4, 6, 2), + ".NETFramework,Version=v4.7" => new(4, 7), + ".NETFramework,Version=v4.7.1" => new(4, 7, 1), + ".NETFramework,Version=v4.7.2" => new(4, 7, 2), + ".NETFramework,Version=v4.8" => new(4, 8), + ".NETFramework,Version=v4.8.1" => new(4, 8, 1), // Null assembly, or TargetFrameworkAttribute not found, or the assembly targeted a version older than we support, // or the assembly targeted a non-framework tfm (like netstandard2.0). _ => null, @@ -64,30 +64,30 @@ internal static string GetFrameworkDescription() return $".NET Framework {releaseVersion} ({servicingVersion})"; } - internal static string GetFrameworkReleaseVersion() + internal static Version GetFrameworkReleaseVersion() { var fullName = System.Runtime.InteropServices.RuntimeInformation.FrameworkDescription; // sth like .NET Framework 4.7.3324.0 var servicingVersion = new string(fullName.SkipWhile(c => !char.IsDigit(c)).ToArray()); return MapToReleaseVersion(servicingVersion); } - internal static string MapToReleaseVersion(string servicingVersion) + internal static Version MapToReleaseVersion(string servicingVersion) { // the following code assumes that .NET 4.6.1 is the oldest supported version if (string.CompareOrdinal(servicingVersion, "4.6.2") < 0) - return "4.6.1"; + return new(4, 6, 1); if (string.CompareOrdinal(servicingVersion, "4.7") < 0) - return "4.6.2"; + return new(4, 6, 2); if (string.CompareOrdinal(servicingVersion, "4.7.1") < 0) - return "4.7"; + return new(4, 7); if (string.CompareOrdinal(servicingVersion, "4.7.2") < 0) - return "4.7.1"; + return new(4, 7, 1); if (string.CompareOrdinal(servicingVersion, "4.8") < 0) - return "4.7.2"; + return new(4, 7, 2); if (string.CompareOrdinal(servicingVersion, "4.8.9") < 0) - return "4.8"; + return new(4, 8); - return "4.8.1"; // most probably the last major release of Full .NET Framework + return new(4, 8, 1); // most probably the last major release of Full .NET Framework } [SupportedOSPlatform("windows")] @@ -101,7 +101,7 @@ internal static string MapToReleaseVersion(string servicingVersion) } [SupportedOSPlatform("windows")] - internal static string? GetLatestNetDeveloperPackVersion() + internal static Version? GetLatestNetDeveloperPackVersion() { if (GetReleaseNumberFromWindowsRegistry() is not int releaseNumber) return null; @@ -112,12 +112,12 @@ internal static string MapToReleaseVersion(string servicingVersion) } // Reference Assemblies exists when Developer Pack is installed - private static bool IsDeveloperPackInstalled(string version) => Directory.Exists(Path.Combine( - ProgramFilesX86DirectoryPath, @"Reference Assemblies\Microsoft\Framework\.NETFramework", 'v' + version)); + private static bool IsDeveloperPackInstalled(Version version) => Directory.Exists(Path.Combine( + ProgramFilesX86DirectoryPath, @"Reference Assemblies\Microsoft\Framework\.NETFramework", $"v{version}")); private static readonly string ProgramFilesX86DirectoryPath = Environment.GetFolderPath( Environment.Is64BitOperatingSystem ? Environment.SpecialFolder.ProgramFilesX86 : Environment.SpecialFolder.ProgramFiles); } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs index 636dcfc35e..d967ee5235 100644 --- a/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs +++ b/src/BenchmarkDotNet/Jobs/EnvironmentMode.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Models; -using BenchmarkDotNet.Portability; using JetBrains.Annotations; namespace BenchmarkDotNet.Jobs @@ -11,7 +10,6 @@ public sealed class EnvironmentMode : JobMode { public static readonly Characteristic PlatformCharacteristic = CreateCharacteristic(nameof(Platform)); public static readonly Characteristic JitCharacteristic = CreateCharacteristic(nameof(Jit)); - public static readonly Characteristic RuntimeCharacteristic = CreateCharacteristic(nameof(Runtime)); public static readonly Characteristic AffinityCharacteristic = CreateCharacteristic(nameof(Affinity)); public static readonly Characteristic GcCharacteristic = CreateCharacteristic(nameof(Gc)); @@ -29,9 +27,6 @@ public sealed class EnvironmentMode : JobMode [PublicAPI] public EnvironmentMode() : this(id: "") { } - [PublicAPI] - public EnvironmentMode(Runtime runtime) : this(runtime.ToString()) => Runtime = runtime; - [PublicAPI] public EnvironmentMode(string id, Jit jit, Platform platform) : this(id) { @@ -63,15 +58,6 @@ public Jit Jit set { JitCharacteristic[this] = value; } } - /// - /// Runtime - /// - public Runtime? Runtime - { - get { return RuntimeCharacteristic[this]; } - set { RuntimeCharacteristic[this] = value; } - } - /// /// ProcessorAffinity for the benchmark process. /// See also: https://msdn.microsoft.com/library/system.diagnostics.process.processoraffinity.aspx @@ -137,17 +123,9 @@ public void SetEnvironmentVariable(EnvironmentVariable variable) EnvironmentVariables = newVariables; } - internal Runtime GetRuntime() - { - return HasValue(RuntimeCharacteristic) && Runtime != null - ? Runtime - : RuntimeInformation.GetCurrentRuntime(); - } - internal BdnEnvironment ToPerfonar() => new() { Jit = HasValue(JitCharacteristic) ? Jit : null, - Runtime = HasValue(RuntimeCharacteristic) ? Runtime?.RuntimeMoniker : null, Affinity = HasValue(AffinityCharacteristic) ? (int)Affinity : null }; } diff --git a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs index 223bd060a2..7ab093d1a3 100644 --- a/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs +++ b/src/BenchmarkDotNet/Jobs/InfrastructureMode.cs @@ -1,36 +1,63 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Models; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; using Perfolizer.Horology; using System.Diagnostics.CodeAnalysis; namespace BenchmarkDotNet.Jobs { - [SuppressMessage("ReSharper", "UnusedMember.Global")] public sealed class InfrastructureMode : JobMode { public const string ReleaseConfigurationName = "Release"; public static readonly Characteristic ToolchainCharacteristic = CreateCharacteristic(nameof(Toolchain)); + public static readonly Characteristic RuntimeCharacteristic = CreateCharacteristic(nameof(Runtime)); public static readonly Characteristic ClockCharacteristic = CreateCharacteristic(nameof(Clock)); public static readonly Characteristic EngineFactoryCharacteristic = CreateCharacteristic(nameof(EngineFactory)); public static readonly Characteristic BuildConfigurationCharacteristic = CreateCharacteristic(nameof(BuildConfiguration)); public static readonly Characteristic> ArgumentsCharacteristic = CreateCharacteristic>(nameof(Arguments)); - public static readonly InfrastructureMode InProcess = new InfrastructureMode(InProcessEmitToolchain.Default); + public static readonly InfrastructureMode InProcess = new(RuntimeInformation.IsAot ? InProcessNoEmitToolchain.Default : InProcessEmitToolchain.Default); public InfrastructureMode() { } private InfrastructureMode(IToolchain toolchain) { + // Go through the property so the coupled runtime characteristic is kept in sync with the toolchain. Toolchain = toolchain; } public IToolchain? Toolchain { get { return ToolchainCharacteristic[this]; } - set { ToolchainCharacteristic[this] = value; } + set + { + ToolchainCharacteristic[this] = value; + // The toolchain and runtime are coupled, and the toolchain is the source of truth for the runtime. + // Setting the toolchain overwrites the runtime with the one the toolchain targets. + RuntimeCharacteristic[this] = value?.Runtime; + } + } + + /// + /// Runtime + /// + public Runtime? Runtime + { + get { return RuntimeCharacteristic[this]; } + set + { + // The toolchain and runtime are coupled. Setting the runtime clears any explicitly set toolchain, + // which is instead derived from the runtime when the benchmark is built. + if (!Equals(RuntimeCharacteristic[this], value)) + ToolchainCharacteristic[this] = null; + RuntimeCharacteristic[this] = value; + } } public IClock? Clock @@ -66,5 +93,10 @@ public bool TryGetToolchain([NotNullWhen(true)] out IToolchain? toolchain) toolchain = HasValue(ToolchainCharacteristic) ? Toolchain : default; return toolchain != default; } + + internal BdnInfrastructure ToPerfonar() => new() + { + Runtime = HasValue(RuntimeCharacteristic) ? Runtime?.ToString() : null + }; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Jobs/JobComparer.cs b/src/BenchmarkDotNet/Jobs/JobComparer.cs index 3a46793219..6359d3fc34 100644 --- a/src/BenchmarkDotNet/Jobs/JobComparer.cs +++ b/src/BenchmarkDotNet/Jobs/JobComparer.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Order; +using System.Collections; namespace BenchmarkDotNet.Jobs { @@ -71,7 +72,46 @@ public int Compare(Job? x, Job? y) return 0; } - public bool Equals(Job? x, Job? y) => Compare(x, y) == 0; + public bool Equals(Job? x, Job? y) + { + if (ReferenceEquals(x, y)) + return true; + if (x is null || y is null) + return false; + if (x.GetType() != y.GetType()) + return false; + + // Compare the actual characteristic values, not their string presentations (which is what Compare uses). + // Presentation is lossy for some characteristics - most importantly a toolchain presents as just its name + // and runtime version, so two jobs that differ only by toolchain settings would otherwise be considered + // equal and silently deduplicated. + foreach (var characteristic in x.GetAllCharacteristics()) + { + // Child-bearing characteristics (the job modes) are already covered by their leaf characteristics. + if (characteristic.HasChildCharacteristics) + continue; + + bool xHasValue = x.HasValue(characteristic); + if (xHasValue != y.HasValue(characteristic)) + return false; + if (xHasValue && !ValuesEqual(characteristic[x], characteristic[y])) + return false; + } + + return true; + } + + private static bool ValuesEqual(object? x, object? y) + { + if (ReferenceEquals(x, y)) + return true; + if (x is null || y is null) + return false; + // Compare collections (e.g. Arguments, EnvironmentVariables) structurally; their elements have value equality. + if (x is not string && x is IEnumerable xItems && y is IEnumerable yItems) + return xItems.Cast().SequenceEqual(yItems.Cast()); + return x.Equals(y); + } public int GetHashCode(Job obj) => obj.Id.GetHashCode(); diff --git a/src/BenchmarkDotNet/Jobs/JobExtensions.cs b/src/BenchmarkDotNet/Jobs/JobExtensions.cs index 2b5e3ff116..1e72e74519 100644 --- a/src/BenchmarkDotNet/Jobs/JobExtensions.cs +++ b/src/BenchmarkDotNet/Jobs/JobExtensions.cs @@ -1,9 +1,9 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.NativeAot; using JetBrains.Annotations; using Perfolizer.Horology; using Perfolizer.Mathematics.OutlierDetection; @@ -19,7 +19,8 @@ public static class JobExtensions // Env public static Job WithJit(this Job job, Jit jit) => job.WithCore(j => j.Environment.Jit = jit); - public static Job WithRuntime(this Job job, Runtime runtime) => job.WithCore(j => j.Environment.Runtime = runtime); + // Runtime and toolchain are coupled; the coupling is enforced by the InfrastructureMode property setters. + public static Job WithRuntime(this Job job, Runtime runtime) => job.WithCore(j => j.Infrastructure.Runtime = runtime); /// /// ProcessorAffinity for the benchmark process. @@ -203,6 +204,7 @@ public static Job WithHeapAffinitizeMask(this Job job, int heapAffinitizeMask) = public static Job WithJitTieringMode(this Job job, JitTieringMode mode) => job.WithCore(j => j.Run.JitTieringMode = mode); // Infrastructure + // Runtime and toolchain are coupled; the coupling is enforced by the InfrastructureMode property setters. public static Job WithToolchain(this Job job, IToolchain toolchain) => job.WithCore(j => j.Infrastructure.Toolchain = toolchain); [PublicAPI] public static Job WithClock(this Job job, IClock clock) => job.WithCore(j => j.Infrastructure.Clock = clock); @@ -337,10 +339,14 @@ internal static Job MakeSettingsUserFriendly(this Job job, Descriptor descriptor return job; } + // The runtime is kept in sync with the toolchain by the InfrastructureMode setters, so we can read it directly. + internal static Runtime GetRuntime(this Job job) + => job.Infrastructure.HasValue(InfrastructureMode.RuntimeCharacteristic) + ? job.Infrastructure.Runtime! + : RuntimeInformation.GetCurrentRuntime(); + internal static bool IsNativeAOT(this Job job) - => job.Environment.GetRuntime() is NativeAotRuntime - // given job can have NativeAOT toolchain set, but Runtime == default - || (job.Infrastructure.TryGetToolchain(out var toolchain) && toolchain is NativeAotToolchain); + => job.GetRuntime() is NativeAotRuntime; private static Job WithCore(this Job job, Action updateCallback) { diff --git a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs index 0354a6b032..56cebf3e92 100644 --- a/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs +++ b/src/BenchmarkDotNet/Loggers/ConsoleLogger.cs @@ -18,7 +18,7 @@ public sealed class ConsoleLogger : ILogger if (Environment.GetEnvironmentVariable("NO_COLOR").IsNotBlank()) return false; - return !(OsDetector.IsAndroid() || OsDetector.IsIOS() || RuntimeInformation.IsWasm || OsDetector.IsTvOS()); + return !(OsDetector.IsMobile() || RuntimeInformation.IsWasm); }); private readonly bool unicodeSupport; diff --git a/src/BenchmarkDotNet/Models/BdnEnvironment.cs b/src/BenchmarkDotNet/Models/BdnEnvironment.cs index 1256d83c6f..f791e635ee 100644 --- a/src/BenchmarkDotNet/Models/BdnEnvironment.cs +++ b/src/BenchmarkDotNet/Models/BdnEnvironment.cs @@ -1,12 +1,10 @@ using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; using Perfolizer.Models; namespace BenchmarkDotNet.Models; internal class BdnEnvironment : EnvironmentInfo { - public RuntimeMoniker? Runtime { get; set; } public Jit? Jit { get; set; } public int? Affinity { get; set; } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Models/BdnInfrastructure.cs b/src/BenchmarkDotNet/Models/BdnInfrastructure.cs new file mode 100644 index 0000000000..557d599587 --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnInfrastructure.cs @@ -0,0 +1,8 @@ +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +internal class BdnInfrastructure : AbstractInfo +{ + public string? Runtime { get; set; } +} diff --git a/src/BenchmarkDotNet/Models/BdnJob.cs b/src/BenchmarkDotNet/Models/BdnJob.cs new file mode 100644 index 0000000000..3af39575a7 --- /dev/null +++ b/src/BenchmarkDotNet/Models/BdnJob.cs @@ -0,0 +1,10 @@ +using Perfolizer.Models; + +namespace BenchmarkDotNet.Models; + +// Extends the perfonar JobInfo (which only exposes Environment and Execution) with an Infrastructure section, +// mirroring how the runtime and toolchain live in Job.Infrastructure. +internal class BdnJob : JobInfo +{ + public BdnInfrastructure? Infrastructure { get; set; } +} diff --git a/src/BenchmarkDotNet/Models/BdnSchema.cs b/src/BenchmarkDotNet/Models/BdnSchema.cs index 3006b702de..d2c26933d3 100644 --- a/src/BenchmarkDotNet/Models/BdnSchema.cs +++ b/src/BenchmarkDotNet/Models/BdnSchema.cs @@ -12,6 +12,7 @@ private BdnSchema() : base("bdn") Add(); Add(); Add(); + Add(); Add(); } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs index e2c51f4f91..ec9cff8a82 100644 --- a/src/BenchmarkDotNet/Portability/RuntimeInformation.cs +++ b/src/BenchmarkDotNet/Portability/RuntimeInformation.cs @@ -48,9 +48,9 @@ internal static class RuntimeInformation #endif #if NETSTANDARD2_0 - public static readonly bool IsAot = IsAotMethod() || FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase); + public static readonly bool IsAot = GetIsAot(); - private static bool IsAotMethod() + private static bool GetIsAot() { Type runtimeFeature = Type.GetType("System.Runtime.CompilerServices.RuntimeFeature"); if (runtimeFeature != null) @@ -63,7 +63,21 @@ private static bool IsAotMethod() } } - return false; + if (FrameworkDescription.StartsWith(".NET Native", StringComparison.OrdinalIgnoreCase)) + { + return true; + } + + // Fallback for old runtimes like legacy MonoAot, test if dynamic method works. + try + { + _ = new System.Reflection.Emit.DynamicMethod("test", typeof(void), []); + return false; + } + catch + { + return true; + } } #else public static readonly bool IsAot = !System.Runtime.CompilerServices.RuntimeFeature.IsDynamicCodeCompiled; @@ -76,9 +90,9 @@ public static bool IsNetCore public static bool IsNativeAOT => Environment.Version.Major >= 5 && IsAot - && !IsWasm && !IsMono; // Wasm and MonoAOTLLVM are also AOT + && !IsWasm && !IsMono; // Wasm and Mono AOT are also AOT - // File-based apps contains specific RuntimeHostConfigurationOptions. + // File-based apps contains specific RuntimeHostConfigurationOptions. // https://github.com/dotnet/dotnet/blob/v10.0.302/src/sdk/documentation/general/dotnet-run-file.md public static bool IsFileBasedApp => AppContext.GetData("EntryPointFilePath") != null && AppContext.GetData("EntryPointFileDirectoryPath") != null; @@ -190,14 +204,13 @@ internal static Runtime GetTargetOrCurrentRuntime(Assembly? assembly) internal static Runtime GetCurrentRuntime() { //do not change the order of conditions because it may cause incorrect determination of runtime - if (IsAot && IsMono) - return MonoAotLLVMRuntime.Default; if (IsWasm) - return WasmRuntime.Default; + return WasmRuntime.GetCurrentVersion(); if (IsNewMono) - return MonoRuntime.GetCurrentVersion(); + // New Mono AOT (MonoAotLLVM) has no official workload, it can only be built by custom runtime artifacts, we don't support it. + return MonoCoreRuntime.GetCurrentVersion(); if (IsOldMono) - return MonoRuntime.Default; + return IsAot ? MonoAotRuntime.Default : MonoRuntime.Default; if (IsFullFramework) return ClrRuntime.GetCurrentVersion(); if (IsNetCore) @@ -205,7 +218,7 @@ internal static Runtime GetCurrentRuntime() if (IsNativeAOT) return NativeAotRuntime.GetCurrentVersion(); - throw new NotSupportedException("Unknown .NET Runtime"); + return UnknownRuntime.Instance; } public static Platform GetCurrentPlatform() @@ -315,5 +328,15 @@ internal static ICollection GetAntivirusProducts() [UnconditionalSuppressMessage(category: "SingleFile", checkId: "IL3000", Justification = "Location property is empty when running with PublishSingleFile/PublishAot")] public static string GetCoreLibDllLocation() => typeof(object).Assembly.Location; + + // Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier() + // returns win10-x64, we want the simpler form win-x64 + // the values taken from https://docs.microsoft.com/en-us/dotnet/core/rid-catalog#macos-rids + internal static string GetPortableRuntimeIdentifier() + { + string osPart = OsDetector.IsWindows() ? "win" : (OsDetector.IsMacOS() ? "osx" : "linux"); + string architecture = ProcessArchitecture.ToString().ToLowerInvariant(); + return $"{osPart}-{architecture}"; + } } } \ No newline at end of file diff --git a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs index aa93984520..24a523d516 100644 --- a/src/BenchmarkDotNet/Properties/AssemblyInfo.cs +++ b/src/BenchmarkDotNet/Properties/AssemblyInfo.cs @@ -9,6 +9,7 @@ [assembly: InternalsVisibleTo("BenchmarkDotNet.Tests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.MonoBenchmarks,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] +[assembly: InternalsVisibleTo("BenchmarkDotNet.IntegrationTests.WasmBenchmarks,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.Windows,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotTrace,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] [assembly: InternalsVisibleTo("BenchmarkDotNet.Diagnostics.dotMemory,PublicKey=" + BenchmarkDotNetInfo.PublicKey)] diff --git a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs index 42485c1920..7475c3fb64 100644 --- a/src/BenchmarkDotNet/Reports/BenchmarkReport.cs +++ b/src/BenchmarkDotNet/Reports/BenchmarkReport.cs @@ -64,9 +64,10 @@ internal EntryInfo ToPerfonar() Parameters = BenchmarkCase.Parameters.PrintInfo, HardwareIntrinsics = this.GetHardwareIntrinsicsInfo() }, - Job = new JobInfo + Job = new BdnJob { Environment = BenchmarkCase.Job.Environment.ToPerfonar(), + Infrastructure = BenchmarkCase.Job.Infrastructure.ToPerfonar(), Execution = BenchmarkCase.Job.Run.ToPerfonar() } }; diff --git a/src/BenchmarkDotNet/Running/BenchmarkCase.cs b/src/BenchmarkDotNet/Running/BenchmarkCase.cs index 4633ecf4f6..ca64cde189 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkCase.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkCase.cs @@ -1,8 +1,12 @@ using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Parameters; using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.InProcess.NoEmit; namespace BenchmarkDotNet.Running { @@ -26,10 +30,24 @@ internal BenchmarkCase(Descriptor descriptor, Job job, ParameterInstances parame Config = config; } - public Runtime GetRuntime() => Job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic) - ? Job.Environment.Runtime! + public Runtime GetRuntime() => Job.Infrastructure.HasValue(InfrastructureMode.RuntimeCharacteristic) + ? Job.Infrastructure.Runtime! : RuntimeInformation.GetTargetOrCurrentRuntime(Descriptor.Type.Assembly); + internal IToolchain GetToolchain() + { + if (Job.Infrastructure.TryGetToolchain(out var toolchain)) + return toolchain; + + // On mobile OSes BDN can't spawn a child process to build/run out of process, so benchmarks must run + // in-process regardless of the runtime (Mono, CoreCLR, NativeAOT): emit IL when dynamic code is + // available, fall back to reflection-only when it isn't (e.g. iOS AOT). + if (OsDetector.IsMobile()) + return RuntimeInformation.IsAot ? InProcessNoEmitToolchain.Default : InProcessEmitToolchain.Default; + + return GetRuntime().GetDefaultToolchain(this); + } + public void Dispose() => Parameters.Dispose(); public int CompareTo(BenchmarkCase? other) diff --git a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs index 2d9d1db0cc..4d48f500b8 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkPartitioner.cs @@ -1,6 +1,5 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Toolchains; namespace BenchmarkDotNet.Running { @@ -28,8 +27,6 @@ public bool Equals(BenchmarkCase? x, BenchmarkCase? y) var jobX = x.Job; var jobY = y.Job; - if (AreDifferent(x.GetRuntime(), y.GetRuntime())) // Mono vs .NET vs Core - return false; if (AreDifferent(x.GetToolchain(), y.GetToolchain())) // Mono vs .NET vs Core vs InProcess return false; if (jobX.Environment.Jit != jobY.Environment.Jit) // Jit is set per exe in .config file @@ -59,7 +56,6 @@ public int GetHashCode(BenchmarkCase obj) { var hashCode = new HashCode(); hashCode.Add(obj.GetToolchain()); - hashCode.Add(obj.GetRuntime()); hashCode.Add(obj.Descriptor.Type.Assembly.Location); hashCode.Add(obj.Descriptor.WorkloadMethod.GetCustomAttributes(false).OfType().Any()); var job = obj.Job; diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 9a744e6ee1..056e5f2e4e 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -108,7 +108,7 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu // .Net SDK 8+ supports ArtifactsPath for proper parallel builds. // Older SDKs may produce builds with incorrect bindings if more than 1 partition is built in parallel. || (partition.RepresentativeBenchmarkCase.GetToolchain().Generator is DotNetCliGenerator - && partition.RepresentativeBenchmarkCase.GetRuntime().RuntimeMoniker.GetRuntimeVersion().Major < 8) + && partition.RepresentativeBenchmarkCase.GetRuntime().Version?.Major < 8) ) .ToArray(); var parallelBuildPartitions = buildPartitions.Except(sequentialBuildPartitions).ToArray(); @@ -289,7 +289,7 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu else if (buildResult.ErrorMessage != null) logger.WriteLineError($"// Build Error: {buildResult.ErrorMessage}"); - if (!benchmark.Job.GetToolchain().IsInProcess) + if (!benchmark.GetToolchain().IsInProcess) { logger.WriteLine(); logger.WriteLineError($"// BenchmarkDotNet has failed to build the auto-generated boilerplate code."); diff --git a/src/BenchmarkDotNet/Running/BuildPartition.cs b/src/BenchmarkDotNet/Running/BuildPartition.cs index 287d879e73..b0f44af5bf 100644 --- a/src/BenchmarkDotNet/Running/BuildPartition.cs +++ b/src/BenchmarkDotNet/Running/BuildPartition.cs @@ -5,10 +5,7 @@ using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.Roslyn; using JetBrains.Annotations; using System.Reflection; @@ -61,18 +58,18 @@ private BuildPartition() [PublicAPI] public Jit Jit => RepresentativeBenchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, Resolver); - public bool IsNativeAot => RepresentativeBenchmarkCase.Job.IsNativeAOT(); + public bool IsNetFramework => Runtime is ClrRuntime; - public bool IsNetFramework => Runtime is ClrRuntime - || (RepresentativeBenchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) && (toolchain is RoslynToolchain || toolchain is CsProjClassicNetToolchain)); - - public Runtime Runtime => RepresentativeBenchmarkCase.Job.Environment.GetRuntime(); + public Runtime Runtime => RepresentativeBenchmarkCase.GetRuntime(); public bool IsCustomBuildConfiguration => BuildConfiguration != InfrastructureMode.ReleaseConfigurationName; - public TimeSpan Timeout => IsNativeAot && RepresentativeBenchmarkCase.Config.BuildTimeout == DefaultConfig.Instance.BuildTimeout - ? TimeSpan.FromMinutes(5) // downloading all NativeAOT dependencies can take a LOT of time - : RepresentativeBenchmarkCase.Config.BuildTimeout; + public TimeSpan Timeout => + // Known slow builds + RepresentativeBenchmarkCase.Job.GetRuntime() is NativeAotRuntime or R2RRuntime or WasmRuntime + && RepresentativeBenchmarkCase.Config.BuildTimeout == DefaultConfig.Instance.BuildTimeout + ? TimeSpan.FromMinutes(5) + : RepresentativeBenchmarkCase.Config.BuildTimeout; public bool LogBuildOutput { get; } @@ -119,11 +116,10 @@ internal bool ForcedNoDependenciesForIntegrationTests if (!XUnitHelper.IsIntegrationTest.Value || !RuntimeInformation.IsNetCore) return false; - var job = RepresentativeBenchmarkCase.Job; - if (job.GetToolchain().Builder is not DotNetCliBuilder) + if (RepresentativeBenchmarkCase.GetToolchain().Builder is not DotNetCliBuilder) return false; - return !job.HasDynamicBuildCharacteristic(); + return !RepresentativeBenchmarkCase.Job.HasDynamicBuildCharacteristic(); } } } diff --git a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt deleted file mode 100644 index f005cd82b4..0000000000 --- a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt +++ /dev/null @@ -1,96 +0,0 @@ - - - $CSPROJPATH$ - $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.props')) - $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.targets')) - - - - - - Exe - False - false - false - $TFM$ - $RUNTIMEPACK$ - $RUNTIMEIDENTIFIER$ - false - false - $PROGRAMNAME$ - false - false - true - BenchmarkDotNet.Autogenerated.UniqueProgramName - true - - - - - - - - - - - - - - - - $COPIEDSETTINGS$ - - - - - latest - - - - - - - $(MicrosoftNetCoreAppRuntimePackDir) - - - - - - - - - - - $([System.IO.Path]::GetFullPath($(PublishDir))) - Dylib - Dll - So - - - - - mcpu=native - - - - - - - - - - - - - - diff --git a/src/BenchmarkDotNet/Templates/WasmCsProj.txt b/src/BenchmarkDotNet/Templates/WasmCsProj.txt index 68915fe4f6..53cc9bd36d 100644 --- a/src/BenchmarkDotNet/Templates/WasmCsProj.txt +++ b/src/BenchmarkDotNet/Templates/WasmCsProj.txt @@ -1,4 +1,4 @@ - + $CSPROJPATH$ $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Wasm.props')) @@ -27,6 +27,8 @@ $CORECLR_OVERRIDES$ false false BenchmarkDotNet.Autogenerated.UniqueProgramName + + $([MSBuild]::NormalizeDirectory('$(MSBuildProjectDirectory)', 'o')) diff --git a/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs b/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs index 28e7ffbd56..60b3099afa 100644 --- a/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/AppConfigGenerator.cs @@ -1,5 +1,4 @@ using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using System.Diagnostics; @@ -38,7 +37,6 @@ internal static async ValueTask GenerateAsync(Job job, TextReader source, TextWr var runtimeElement = GetOrCreateRuntimeElement(configurationElement); - ClearStartupSettingsForCustomClr(configurationElement, job.Environment.Runtime); ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(runtimeElement); GenerateJitSettings(runtimeElement, job.Environment); @@ -103,20 +101,6 @@ private static void ClearAllRuntimeSettingsThatCanBeSetOnlyByJobConfiguration(XE toRemove?.ForEach(e => e.Remove()); } - private static void ClearStartupSettingsForCustomClr(XElement configurationElement, Runtime? runtime) - { - if (!(runtime is ClrRuntime clrRuntime) || clrRuntime.Version.IsBlank()) - return; - - List? toRemove = null; - foreach (var child in configurationElement.Elements("startup")) - { - toRemove ??= new List(); - toRemove.Add(child); - } - toRemove?.ForEach(e => e.Remove()); - } - private static void GenerateJitSettings(XElement runtimeElement, EnvironmentMode environmentMode) { if (environmentMode.HasValue(EnvironmentMode.JitCharacteristic)) diff --git a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs index 904fd3ffb6..d600cf1dcc 100644 --- a/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs +++ b/src/BenchmarkDotNet/Toolchains/ArtifactsPaths.cs @@ -1,51 +1,34 @@ using BenchmarkDotNet.Extensions; -using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains { - public class ArtifactsPaths + public class ArtifactsPaths( + string rootArtifactsFolderPath, + string buildArtifactsDirectoryPath, + string binariesDirectoryPath, + string publishDirectoryPath, + string programCodePath, + string appConfigPath, + string nuGetConfigPath, + string projectFilePath, + string buildScriptFilePath, + string executablePath, + string programName, + string packagesDirectoryName) { public static readonly ArtifactsPaths Empty = new("", "", "", "", "", "", "", "", "", "", "", ""); - [PublicAPI] public string RootArtifactsFolderPath { get; } - [PublicAPI] public string BuildArtifactsDirectoryPath { get; } - [PublicAPI] public string BinariesDirectoryPath { get; } - [PublicAPI] public string PublishDirectoryPath { get; } - [PublicAPI] public string ProgramCodePath { get; } - [PublicAPI] public string AppConfigPath { get; } - [PublicAPI] public string NuGetConfigPath { get; } - [PublicAPI] public string ProjectFilePath { get; } - [PublicAPI] public string BuildScriptFilePath { get; } - [PublicAPI] public string ExecutablePath { get; } - [PublicAPI] public string ProgramName { get; } - [PublicAPI] public string PackagesDirectoryName { get; } - - public ArtifactsPaths( - string rootArtifactsFolderPath, - string buildArtifactsDirectoryPath, - string binariesDirectoryPath, - string publishDirectoryPath, - string programCodePath, - string appConfigPath, - string nuGetConfigPath, - string projectFilePath, - string buildScriptFilePath, - string executablePath, - string programName, - string packagesDirectoryName) - { - RootArtifactsFolderPath = rootArtifactsFolderPath; - BuildArtifactsDirectoryPath = buildArtifactsDirectoryPath; - BinariesDirectoryPath = binariesDirectoryPath; - PublishDirectoryPath = publishDirectoryPath.EnsureNotNull(); - ProgramCodePath = programCodePath.EnsureNotNull(); - AppConfigPath = appConfigPath.EnsureNotNull(); - NuGetConfigPath = nuGetConfigPath.EnsureNotNull(); - ProjectFilePath = projectFilePath.EnsureNotNull(); - BuildScriptFilePath = buildScriptFilePath.EnsureNotNull(); - ExecutablePath = executablePath; - ProgramName = programName; - PackagesDirectoryName = packagesDirectoryName.EnsureNotNull(); - } + public string RootArtifactsFolderPath { get; } = rootArtifactsFolderPath; + public string BuildArtifactsDirectoryPath { get; } = buildArtifactsDirectoryPath; + public string BinariesDirectoryPath { get; } = binariesDirectoryPath; + public string PublishDirectoryPath { get; } = publishDirectoryPath.EnsureNotNull(); + public string ProgramCodePath { get; } = programCodePath.EnsureNotNull(); + public string AppConfigPath { get; } = appConfigPath.EnsureNotNull(); + public string NuGetConfigPath { get; } = nuGetConfigPath.EnsureNotNull(); + public string ProjectFilePath { get; } = projectFilePath.EnsureNotNull(); + public string BuildScriptFilePath { get; } = buildScriptFilePath.EnsureNotNull(); + public string ExecutablePath { get; } = executablePath; + public string ProgramName { get; } = programName; + public string PackagesDirectoryName { get; } = packagesDirectoryName.EnsureNotNull(); } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunGenerator.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunGenerator.cs index bb245a796a..cb43b5d699 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunGenerator.cs @@ -4,8 +4,8 @@ namespace BenchmarkDotNet.Toolchains.CoreRun { public class CoreRunGenerator : CsProjGenerator { - public CoreRunGenerator(FileInfo sourceCoreRun, FileInfo copyCoreRun, string targetFrameworkMoniker, string cliPath, string packagesPath) - : base(targetFrameworkMoniker, cliPath, packagesPath) + public CoreRunGenerator(FileInfo sourceCoreRun, FileInfo copyCoreRun, CoreRunSettings settings) + : base(settings) { SourceCoreRun = sourceCoreRun; CopyCoreRun = copyCoreRun; @@ -17,10 +17,8 @@ public CoreRunGenerator(FileInfo sourceCoreRun, FileInfo copyCoreRun, string tar private bool NeedsCopy => SourceCoreRun != CopyCoreRun; - protected override string GetPackagesDirectoryPath(string buildArtifactsDirectoryPath) => PackagesPath; - protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, "publish"); + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, "publish"); protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths) { diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs index c932bf66bf..bc24b3970e 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunPublisher.cs @@ -6,7 +6,7 @@ namespace BenchmarkDotNet.Toolchains.CoreRun { - public class CoreRunPublisher(string tfm, FileInfo coreRun, FileInfo? customDotNetCliPath = null) : DotNetCliPublisher(tfm, customDotNetCliPath?.FullName ?? "") + public class CoreRunPublisher(DotNetCliSettings settings, FileInfo coreRun) : DotNetCliPublisher(settings) { public override async ValueTask BuildAsync(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger, CancellationToken cancellationToken) { diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunSettings.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunSettings.cs new file mode 100644 index 0000000000..f0c1248e30 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunSettings.cs @@ -0,0 +1,47 @@ +using System.Collections.Generic; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.CoreRun; + +/// Settings for the . +public sealed record CoreRunSettings : DotNetCliSettings +{ + /// + /// Path to CoreRun.exe (corerun on Unix). BDN expects the path to CoreRun itself, not the Core_Root folder. + /// + public required FileInfo SourceCoreRun { get; init; } + + /// + /// Whether to shadow-copy CoreRun (true by default). The toolchain replaces old dependencies in the CoreRun + /// folder with newer versions used by the benchmarks, so copying avoids mutating the original. + /// + public bool CreateCopy { get; init; } = true; + + /// Display name for the toolchain ("CoreRun" by default). + public string DisplayName { get; init; } = "CoreRun"; + + public CoreRunSettings() { } + + internal CoreRunSettings(CommandLineOptions options) : base(options) { } + + /// + public override void FillSettings(IDictionary settings) + { + base.FillSettings(settings); + settings[nameof(SourceCoreRun)] = SourceCoreRun.FullName; + settings[nameof(CreateCopy)] = CreateCopy; + settings[nameof(DisplayName)] = DisplayName; + } + + // FileInfo compares by reference; compare SourceCoreRun by value and chain into the base record. + public bool Equals(CoreRunSettings? other) + => other is not null + && base.Equals(other) + && SourceCoreRun?.FullName == other.SourceCoreRun?.FullName + && CreateCopy == other.CreateCopy + && DisplayName == other.DisplayName; + + public override int GetHashCode() + => HashCode.Combine(base.GetHashCode(), SourceCoreRun?.FullName, CreateCopy, DisplayName); +} diff --git a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs index c372051e02..9382be944e 100644 --- a/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/CoreRun/CoreRunToolchain.cs @@ -1,42 +1,52 @@ using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Toolchains.CoreRun { - public class CoreRunToolchain : IToolchain + public sealed class CoreRunToolchain : IToolchain, IHasSettings { - /// - /// creates a CoreRunToolchain which is using provided CoreRun to execute .NET Core apps - /// - /// the path to CoreRun - /// /should a copy of CoreRun be performed? True by default. The toolchain replaces old dependencies in CoreRun folder with newer versions if used by the benchmarks. - /// TFM, net10.0 is the default - /// path to dotnet cli, if not provided the one from PATH will be used - /// display name, CoreRun is the default value - /// the directory to restore packages to - public CoreRunToolchain(FileInfo coreRun, bool createCopy = true, - string targetFrameworkMoniker = "net10.0", - FileInfo? customDotNetCliPath = null, - DirectoryInfo? restorePath = null, - string displayName = "CoreRun") + private const string DefaultTargetFrameworkMoniker = "net11.0"; + + private CoreRunToolchain(CoreRunSettings settings) { - if (!coreRun.Exists) + if (!settings.SourceCoreRun.Exists) throw new FileNotFoundException("Provided CoreRun path does not exist. Please remember that BDN expects path to CoreRun.exe (corerun on Unix), not to Core_Root folder."); - SourceCoreRun = coreRun; - CopyCoreRun = createCopy ? GetShadowCopyPath(coreRun) : coreRun; - CustomDotNetCliPath = customDotNetCliPath; - RestorePath = restorePath; - - Name = displayName; - Generator = new CoreRunGenerator(SourceCoreRun, CopyCoreRun, targetFrameworkMoniker, customDotNetCliPath?.FullName ?? "", restorePath?.FullName ?? ""); - Builder = new CoreRunPublisher(targetFrameworkMoniker, CopyCoreRun, customDotNetCliPath); - Executor = new DotNetCliExecutor(customDotNetCliPath: CopyCoreRun.FullName); // instead of executing "dotnet $pathToDll" we do "CoreRun $pathToDll" + Settings = settings; + SourceCoreRun = settings.SourceCoreRun; + CopyCoreRun = settings.CreateCopy ? GetShadowCopyPath(settings.SourceCoreRun) : settings.SourceCoreRun; + + // The build components receive the resolved settings (target framework moniker filled in); the original + // `settings` is stored in Settings for equality and the settings column. + var resolvedSettings = Resolve(settings, DefaultTargetFrameworkMoniker); + // Parsed rather than picked apart by hand: neither a platform suffix (net10.0-windows) nor a + // netcoreappX.Y survives stripping "net" and calling Version.Parse. + Runtime = Environments.Runtime.TryParse(resolvedSettings.TargetFrameworkMoniker, out var parsed) && parsed is CoreRuntime coreRuntime + ? coreRuntime + : throw new NotSupportedException( + $"CoreRun can only run .NET (Core) benchmarks, but '{resolvedSettings.TargetFrameworkMoniker}' does not describe a .NET (Core) target framework."); + Generator = new CoreRunGenerator(SourceCoreRun, CopyCoreRun, resolvedSettings); + Builder = new CoreRunPublisher(resolvedSettings, CopyCoreRun); + Executor = new DotNetCliExecutor(customDotNetCliPath: CopyCoreRun); // instead of executing "dotnet $pathToDll" we do "CoreRun $pathToDll" } - public string Name { get; } + /// Returns a toolchain that uses the provided CoreRun to execute .NET Core apps. + public static CoreRunToolchain From(CoreRunSettings settings) => new(settings); + + // Fills the target framework moniker in with the default only when the user left it unset, avoiding the + // settings copy otherwise. CoreRun has no runtime to derive it from - it parses the runtime out of the moniker. + private static CoreRunSettings Resolve(CoreRunSettings settings, string fallbackTfm) + => settings.TargetFrameworkMoniker.IsNotBlank() ? settings : settings with { TargetFrameworkMoniker = fallbackTfm }; + + internal CoreRunSettings Settings { get; } + + ISettings IHasSettings.Settings => Settings; + + public Runtime Runtime { get; } public IGenerator Generator { get; } @@ -50,11 +60,14 @@ public CoreRunToolchain(FileInfo coreRun, bool createCopy = true, public FileInfo CopyCoreRun { get; } - public FileInfo? CustomDotNetCliPath { get; } + public override string ToString() => $"{Settings.DisplayName} {Runtime.Version}"; - public DirectoryInfo? RestorePath { get; } + public override bool Equals(object? obj) + => obj is CoreRunToolchain other + && Runtime.Equals(other.Runtime) + && Settings.Equals(other.Settings); - public override string ToString() => Name; + public override int GetHashCode() => HashCode.Combine(Runtime, Settings); public async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmark, IResolver resolver) { @@ -64,7 +77,7 @@ public async IAsyncEnumerable ValidateAsync(BenchmarkCase bench $"Provided CoreRun path does not exist, benchmark '{benchmark.DisplayInfo}' will not be executed. Please remember that BDN expects path to CoreRun.exe (corerun on Unix), not to Core_Root folder.", benchmark); } - else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath?.FullName, benchmark, out var invalidCliError)) + else if (DotNetSdkValidator.IsCliPathInvalid(Settings.CliPath, benchmark, out var invalidCliError)) { yield return invalidCliError; } @@ -111,4 +124,4 @@ static bool TryToCreateSubfolder(DirectoryInfo directory) } } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs deleted file mode 100644 index 94cfefbffa..0000000000 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjClassicNetToolchain.cs +++ /dev/null @@ -1,73 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Detectors; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Validators; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.CsProj -{ - /// - /// this toolchain is designed for the new .csprojs, to build .NET 4.x benchmarks from the context of .NET Core host process - /// it does not work with the old .csprojs or project.json! - /// - [PublicAPI] - public class CsProjClassicNetToolchain : Toolchain - { - [PublicAPI] public static readonly IToolchain Net461 = new CsProjClassicNetToolchain("net461", ".NET Framework 4.6.1"); - [PublicAPI] public static readonly IToolchain Net462 = new CsProjClassicNetToolchain("net462", ".NET Framework 4.6.2"); - [PublicAPI] public static readonly IToolchain Net47 = new CsProjClassicNetToolchain("net47", ".NET Framework 4.7"); - [PublicAPI] public static readonly IToolchain Net471 = new CsProjClassicNetToolchain("net471", ".NET Framework 4.7.1"); - [PublicAPI] public static readonly IToolchain Net472 = new CsProjClassicNetToolchain("net472", ".NET Framework 4.7.2"); - [PublicAPI] public static readonly IToolchain Net48 = new CsProjClassicNetToolchain("net48", ".NET Framework 4.8"); - [PublicAPI] public static readonly IToolchain Net481 = new CsProjClassicNetToolchain("net481", ".NET Framework 4.8.1"); - - internal string CustomDotNetCliPath { get; } - - private CsProjClassicNetToolchain(string targetFrameworkMoniker, string name, string packagesPath = "", string customDotNetCliPath = "") - : base(name, - new CsProjGenerator( - targetFrameworkMoniker, - customDotNetCliPath, - packagesPath, - isNetCore: false), - new DotNetCliBuilder(targetFrameworkMoniker, customDotNetCliPath), - new Executor()) - { - CustomDotNetCliPath = customDotNetCliPath; - } - - public static IToolchain From(string targetFrameworkMoniker, string packagesPath = "", string customDotNetCliPath = "") - => new CsProjClassicNetToolchain( - targetFrameworkMoniker, - name: targetFrameworkMoniker, - packagesPath.EnsureNotNull(), - customDotNetCliPath.EnsureNotNull()); - - public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) - { - await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) - { - yield return validationError; - } - - if (!OsDetector.IsWindows()) - { - yield return new ValidationError(true, - $"Classic .NET toolchain is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - yield break; - } - else if (DotNetSdkValidator.IsCliPathInvalid(CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) - { - yield return invalidCliError; - } - - foreach (var validationError in DotNetSdkValidator.ValidateFrameworkSdks(benchmarkCase)) - { - yield return validationError; - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs deleted file mode 100644 index e53d5ef979..0000000000 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjCoreToolchain.cs +++ /dev/null @@ -1,97 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.InProcess.Emit; -using BenchmarkDotNet.Validators; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.CsProj -{ - [PublicAPI] - public class CsProjCoreToolchain : Toolchain, IEquatable - { - [PublicAPI] public static readonly IToolchain NetCoreApp20 = From(NetCoreAppSettings.NetCoreApp20); - [PublicAPI] public static readonly IToolchain NetCoreApp21 = From(NetCoreAppSettings.NetCoreApp21); - [PublicAPI] public static readonly IToolchain NetCoreApp22 = From(NetCoreAppSettings.NetCoreApp22); - [PublicAPI] public static readonly IToolchain NetCoreApp30 = From(NetCoreAppSettings.NetCoreApp30); - [PublicAPI] public static readonly IToolchain NetCoreApp31 = From(NetCoreAppSettings.NetCoreApp31); - [PublicAPI] public static readonly IToolchain NetCoreApp50 = From(NetCoreAppSettings.NetCoreApp50); - [PublicAPI] public static readonly IToolchain NetCoreApp60 = From(NetCoreAppSettings.NetCoreApp60); - [PublicAPI] public static readonly IToolchain NetCoreApp70 = From(NetCoreAppSettings.NetCoreApp70); - [PublicAPI] public static readonly IToolchain NetCoreApp80 = From(NetCoreAppSettings.NetCoreApp80); - [PublicAPI] public static readonly IToolchain NetCoreApp90 = From(NetCoreAppSettings.NetCoreApp90); - [PublicAPI] public static readonly IToolchain NetCoreApp10_0 = From(NetCoreAppSettings.NetCoreApp10_0); - [PublicAPI] public static readonly IToolchain NetCoreApp11_0 = From(NetCoreAppSettings.NetCoreApp11_0); - - internal CsProjCoreToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) - : base(name, generator, builder, executor) - { - CustomDotNetCliPath = customDotNetCliPath; - } - - internal string CustomDotNetCliPath { get; } - - [PublicAPI] - public static IToolchain From(NetCoreAppSettings settings) - => new CsProjCoreToolchain(settings.Name, - new CsProjGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, settings.RuntimeFrameworkVersion), - new DotNetCliBuilder(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath), - new DotNetCliExecutor(settings.CustomDotNetCliPath), - settings.CustomDotNetCliPath); - - public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) - { - await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) - { - yield return validationError; - } - - if (benchmarkCase.Job.HasValue(EnvironmentMode.JitCharacteristic) && benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.LegacyJit) - { - yield return new ValidationError(true, - $"Currently dotnet cli toolchain supports only RyuJit, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - } - if (benchmarkCase.Job.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver)) - { - yield return new ValidationError(true, - $"Currently project.json does not support CpuGroups (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - } - if (benchmarkCase.Job.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver)) - { - yield return new ValidationError(true, - $"Currently project.json does not support gcAllowVeryLargeObjects (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - } - - var benchmarkAssembly = benchmarkCase.Descriptor.Type.Assembly; - if (benchmarkAssembly.IsLinqPad()) - { - yield return new ValidationError(true, - $"Currently CsProjCoreToolchain does not support LINQPad 6+. Please use {nameof(InProcessEmitToolchain)} instead.", - benchmarkCase); - } - - foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) - { - yield return validationError; - } - } - - public override bool Equals(object? obj) => obj is CsProjCoreToolchain typed && Equals(typed); - - public bool Equals(CsProjCoreToolchain? other) - { - if (other == null) - return false; - - return Generator.Equals(other.Generator); - } - - public override int GetHashCode() => Generator.GetHashCode(); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs index 08956ba6ea..16ddc27e11 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs @@ -15,13 +15,13 @@ namespace BenchmarkDotNet.Toolchains.CsProj { - [PublicAPI] - public class CsProjGenerator : DotNetCliGenerator, IEquatable + public class CsProjGenerator(DotNetCliSettings settings, bool isNetCore = true) + : DotNetCliGenerator(settings, isNetCore) { private const string DefaultSdkName = "Microsoft.NET.Sdk"; - private static readonly ImmutableArray SettingsWeWantToCopy = new[] - { + private static readonly ImmutableArray SettingsWeWantToCopy = + [ "NetCoreAppImplicitPackageVersion", "RuntimeFrameworkVersion", "PackageTargetFallback", @@ -45,21 +45,13 @@ public class CsProjGenerator : DotNetCliGenerator, IEquatable "SuppressTfmSupportBuildWarnings", "GarbageCollectionAdaptationMode", // TODO: Remove this setting after https://github.com/dotnet/runtime/pull/131069 issue is resolved, and backported to .NET 10. - }.ToImmutableArray(); - - public string RuntimeFrameworkVersion { get; } - - public CsProjGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string runtimeFrameworkVersion = "", bool isNetCore = true) - : base(targetFrameworkMoniker, cliPath, packagesPath, isNetCore) - { - RuntimeFrameworkVersion = runtimeFrameworkVersion.EnsureNotNull(); - } + ]; protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) { string assemblyLocation = buildPartition.RepresentativeBenchmarkCase.Descriptor.Type.Assembly.Location; - //Assembles loaded from a stream will have an empty location (https://docs.microsoft.com/en-us/dotnet/api/system.reflection.assembly.location). + //Assemblies loaded from a stream will have an empty location (https://docs.microsoft.com/en-us/dotnet/api/system.reflection.assembly.location). string directoryName = assemblyLocation.IsEmpty() ? Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Bin") : Path.GetDirectoryName(buildPartition.AssemblyLocation)!; @@ -71,17 +63,18 @@ protected override string GetProjectFilePath(string buildArtifactsDirectoryPath) => Path.Combine(buildArtifactsDirectoryPath, "BenchmarkDotNet.Autogenerated.csproj"); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker); + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker); protected override ValueTask GenerateBuildScriptAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) { string projectFilePath = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, NullLogger.Instance).FullName; + string cli = Settings.CliPath?.FullName ?? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value; var content = new StringBuilder(300) - .AppendLine($"call {CliPath} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, projectFilePath)}") - .AppendLine($"call {CliPath} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, projectFilePath, TargetFrameworkMoniker)}") - .AppendLine($"call {CliPath} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath)}") - .AppendLine($"call {CliPath} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, TargetFrameworkMoniker)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, projectFilePath)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, projectFilePath, Settings.TargetFrameworkMoniker)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, Settings.TargetFrameworkMoniker)}") .ToString(); return new(File.WriteAllTextAsync(artifactsPaths.BuildScriptFilePath, content, cancellationToken)); @@ -116,7 +109,7 @@ private async ValueTask GenerateBuildProject(BuildPartition buildPartiti .Replace("$PLATFORM$", buildPartition.Platform.ToConfig()) .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) .Replace("$CSPROJPATH$", projectFile.FullName) - .Replace("$TFM$", TargetFrameworkMoniker) + .Replace("$TFM$", Settings.TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$RUNTIMESETTINGS$", GetRuntimeSettings(benchmark.Job.Environment.Gc, buildPartition.Resolver)) .Replace("$COPIEDSETTINGS$", customProperties) @@ -163,9 +156,9 @@ public static int Main(string[] args) // Build the original project then reference all of the built dlls. BuildResult buildResult = await new DotNetCliCommand( - CliPath, + Settings.CliPath, gathererProject, - TargetFrameworkMoniker, + Settings.TargetFrameworkMoniker, arguments: "", GenerateResult.Success(artifactsPaths, []), logger, @@ -197,7 +190,7 @@ public static int Main(string[] args) itemGroup.Add(new XElement("Reference", new XAttribute("Include", Path.GetFileNameWithoutExtension(assemblyFile)), new XElement("HintPath", assemblyFile) - // TODO: Add Aliases here for extern alias #2289 + // TODO: Add Aliases here for extern alias #2289 )); } @@ -229,21 +222,14 @@ protected virtual string GetRuntimeSettings(GcMode gcMode, IResolver resolver) // the host project or one of the .props file that it imports might contain some custom settings that needs to be copied, sth like // 2.0.0-beta-001607-00 // 2.0.0-beta-001607-00 - internal (string customProperties, string sdkName) GetSettingsThatNeedToBeCopied(XmlDocument xmlDoc, FileInfo projectFile) + protected internal (string customProperties, string sdkName) GetSettingsThatNeedToBeCopied(XmlDocument xmlDoc, FileInfo projectFile) { - if (RuntimeFrameworkVersion.IsNotBlank()) // some power users knows what to configure, just do it and copy nothing more - { - return (@$" - {RuntimeFrameworkVersion} - ", DefaultSdkName); - } - XmlElement projectElement = xmlDoc.DocumentElement!; // custom SDKs are not added for non-netcoreapp apps (like net471), so when the TFM != netcoreapp we dont parse " string? sdkName = null; - if (TargetFrameworkMoniker.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase)) + if (Settings.TargetFrameworkMoniker.StartsWith("netcoreapp", StringComparison.InvariantCultureIgnoreCase)) { foreach (XmlElement importElement in projectElement.GetElementsByTagName("Import")) { @@ -378,25 +364,6 @@ protected virtual FileInfo GetProjectFilePath(Type benchmarkTarget, ILogger logg var projectFile = Helpers.FindProjectFile(rootDirectory, projectName); return projectFile; } - - public override bool Equals(object? obj) => obj is CsProjGenerator other && Equals(other); - - public bool Equals(CsProjGenerator? other) - { - if (ReferenceEquals(this, other)) - return true; - - if (other is null) - return false; - - return TargetFrameworkMoniker == other.TargetFrameworkMoniker - && RuntimeFrameworkVersion == other.RuntimeFrameworkVersion - && CliPath == other.CliPath - && PackagesPath == other.PackagesPath; - } - - public override int GetHashCode() - => HashCode.Combine(TargetFrameworkMoniker, RuntimeFrameworkVersion, CliPath, PackagesPath); } internal static class Helpers diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjNetToolchain.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjNetToolchain.cs new file mode 100644 index 0000000000..63335ab951 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjNetToolchain.cs @@ -0,0 +1,65 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Toolchains.CsProj; + +public abstract class CsProjNetToolchain(string name, Runtime runtime, DotNetCliSettings settings, IGenerator generator, IBuilder builder, IExecutor executor) + : Toolchain(name, runtime, generator, builder, executor), IHasSettings +{ + internal DotNetCliSettings Settings => settings; + ISettings IHasSettings.Settings => Settings; + + public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) + { + await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) + { + yield return validationError; + } + + if (benchmarkCase.Job.HasValue(EnvironmentMode.JitCharacteristic) && benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.LegacyJit) + { + yield return new ValidationError(true, + $"{GetType().Name} supports only RyuJit, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + } + if (benchmarkCase.Job.ResolveValue(GcMode.CpuGroupsCharacteristic, resolver)) + { + yield return new ValidationError(true, + $"Currently {GetType().Name} does not support CpuGroups (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + } + if (benchmarkCase.Job.ResolveValue(GcMode.AllowVeryLargeObjectsCharacteristic, resolver)) + { + yield return new ValidationError(true, + $"Currently {GetType().Name} does not support gcAllowVeryLargeObjects (app.config does), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + } + + var benchmarkAssembly = benchmarkCase.Descriptor.Type.Assembly; + if (benchmarkAssembly.IsLinqPad()) + { + yield return new ValidationError(true, + $"Currently {GetType().Name} does not support LINQPad 6+. Please use {nameof(InProcessEmitToolchain)} instead.", + benchmarkCase); + } + + foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(settings.CliPath, benchmarkCase)) + { + yield return validationError; + } + } + + public override bool Equals(object? obj) + => obj is CsProjNetToolchain other + && other.GetType() == GetType() + && Runtime.Equals(other.Runtime) + && Settings.Equals(other.Settings); + + public override int GetHashCode() => HashCode.Combine(GetType(), Runtime, Settings); +} diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs deleted file mode 100644 index a685f443c7..0000000000 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/CustomDotNetCliToolchainBuilder.cs +++ /dev/null @@ -1,135 +0,0 @@ -using System.Diagnostics.CodeAnalysis; -using System.Runtime.InteropServices; -using BenchmarkDotNet.Detectors; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.DotNetCli -{ - [SuppressMessage("ReSharper", "InconsistentNaming")] - public abstract class CustomDotNetCliToolchainBuilder - { - protected readonly Dictionary Feeds = []; - - protected string? runtimeIdentifier; - protected string? customDotNetCliPath; - protected string? displayName; - protected string? runtimeFrameworkVersion; - - protected bool useNuGetClearTag; - protected bool useTempFolderForRestore; - private string? targetFrameworkMoniker; - - public abstract IToolchain ToToolchain(); - - /// it allows you to define an additional NuGet feed, you can seal the feeds list by using the UseNuGetClearTag() method - /// the name of the feed, will be used in the auto-generated NuGet.config file - /// the address of the feed, will be used in the auto-generated NuGet.config file - [PublicAPI] - public CustomDotNetCliToolchainBuilder AdditionalNuGetFeed(string feedName, string feedAddress) - { - if (string.IsNullOrEmpty(feedName)) throw new ArgumentException("Value cannot be null or empty.", nameof(feedName)); - if (string.IsNullOrEmpty(feedAddress)) throw new ArgumentException("Value cannot be null or empty.", nameof(feedAddress)); - - Feeds[feedName] = feedAddress; - - return this; - } - - /// - /// emits clear tag in the auto-generated NuGet.config file - /// - public CustomDotNetCliToolchainBuilder UseNuGetClearTag(bool value) - { - useNuGetClearTag = value; - - return this; - } - - /// TFM, example: net10.0 - [PublicAPI] - [SuppressMessage("ReSharper", "ParameterHidesMember")] - public CustomDotNetCliToolchainBuilder TargetFrameworkMoniker(string targetFrameworkMoniker) - { - this.targetFrameworkMoniker = targetFrameworkMoniker ?? throw new ArgumentNullException(nameof(targetFrameworkMoniker)); - - return this; - } - - protected string GetTargetFrameworkMoniker() - { - if (targetFrameworkMoniker.IsNotBlank()) - return targetFrameworkMoniker; - if (!Portability.RuntimeInformation.IsNetCore) - throw new NotSupportedException("You must specify the target framework moniker in explicit way using builder.TargetFrameworkMoniker(tfm) method"); - - return CoreRuntime.GetCurrentVersion().MsBuildMoniker; - } - - /// if not provided, the one from PATH will be used - [PublicAPI] - public CustomDotNetCliToolchainBuilder DotNetCli(string newCustomDotNetCliPath) - { - if (newCustomDotNetCliPath.IsNotBlank() && !File.Exists(newCustomDotNetCliPath)) - throw new FileNotFoundException("Given file does not exist", newCustomDotNetCliPath); - - customDotNetCliPath = newCustomDotNetCliPath; - - return this; - } - - /// if not provided, portable OS-arch will be used (example: "win-x64", "linux-x86") - [PublicAPI] - public CustomDotNetCliToolchainBuilder RuntimeIdentifier(string newRuntimeIdentifier) - { - runtimeIdentifier = newRuntimeIdentifier; - - return this; - } - - /// optional, when set it's copied to the generated .csproj file - [PublicAPI] - public CustomDotNetCliToolchainBuilder RuntimeFrameworkVersion(string newRuntimeFrameworkVersion) - { - runtimeFrameworkVersion = newRuntimeFrameworkVersion; - - return this; - } - - /// the name of the toolchain to be displayed in results - [PublicAPI] - public CustomDotNetCliToolchainBuilder DisplayName(string newDisplayName) - { - if (string.IsNullOrEmpty(newDisplayName)) throw new ArgumentException("Value cannot be null or empty.", nameof(newDisplayName)); - - displayName = newDisplayName; - - return this; - } - - /// - /// restore to temp folder to keep your CI clean or install same package many times (perhaps with different content but same version number), by default true for local builds - /// https://github.com/dotnet/corefx/blob/master/Documentation/project-docs/dogfooding.md#3---consuming-subsequent-code-changes-by-rebuilding-the-package-alternative-2 - /// - [PublicAPI] - public CustomDotNetCliToolchainBuilder UseTempFolderForRestore(bool value) - { - useTempFolderForRestore = value; - - return this; - } - - internal static string GetPortableRuntimeIdentifier() - { - // Microsoft.DotNet.PlatformAbstractions.RuntimeEnvironment.GetRuntimeIdentifier() - // returns win10-x64, we want the simpler form win-x64 - // the values taken from https://docs.microsoft.com/en-us/dotnet/core/rid-catalog#macos-rids - string osPart = OsDetector.IsWindows() ? "win" : (OsDetector.IsMacOS() ? "osx" : "linux"); - - string architecture = RuntimeInformation.ProcessArchitecture.ToString().ToLowerInvariant(); - - return $"{osPart}-{architecture}"; - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs index 67045ed3e2..eabf2ea82e 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliBuilder.cs @@ -1,39 +1,27 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; -using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.DotNetCli { - [PublicAPI] - public class DotNetCliBuilder : IBuilder + // The toolchain resolves the target framework moniker against the runtime before constructing the builder. + public class DotNetCliBuilder(DotNetCliSettings settings, bool logOutput = false) : IBuilder { - private string TargetFrameworkMoniker { get; } - - private string CustomDotNetCliPath { get; } - private bool LogOutput { get; } - - [PublicAPI] - public DotNetCliBuilder(string targetFrameworkMoniker, string customDotNetCliPath = "", bool logOutput = false) - { - TargetFrameworkMoniker = targetFrameworkMoniker; - CustomDotNetCliPath = customDotNetCliPath; - LogOutput = logOutput; - } + internal FileInfo? CustomDotNetCliPath { get; } = settings.CliPath; public async ValueTask BuildAsync(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger, CancellationToken cancellationToken) { var buildResult = await new DotNetCliCommand( CustomDotNetCliPath, generateResult.ArtifactsPaths.ProjectFilePath, - TargetFrameworkMoniker, + settings.TargetFrameworkMoniker, string.Empty, generateResult, logger, buildPartition, [], buildPartition.Timeout, - logOutput: LogOutput + logOutput: logOutput ) .RestoreThenBuildAsync(cancellationToken) .ConfigureAwait(false); diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index 01005e450f..5c7493fe01 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -12,13 +12,13 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli { public class DotNetCliCommand { - [PublicAPI] public string CliPath { get; } + [PublicAPI] public FileInfo? CliPath { get; } [PublicAPI] public string FilePath { get; } [PublicAPI] public string TargetFrameworkMoniker { get; } - [PublicAPI] public string Arguments { get; } + [PublicAPI] public string? Arguments { get; } [PublicAPI] public GenerateResult GenerateResult { get; } @@ -32,10 +32,10 @@ public class DotNetCliCommand [PublicAPI] public bool LogOutput { get; } - public DotNetCliCommand(string cliPath, string filePath, string tfm, string arguments, GenerateResult generateResult, ILogger logger, + public DotNetCliCommand(FileInfo? cliPath, string filePath, string tfm, string? arguments, GenerateResult generateResult, ILogger logger, BuildPartition buildPartition, IReadOnlyList environmentVariables, TimeSpan timeout, bool logOutput = false) { - CliPath = cliPath.IsBlank() ? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value : cliPath; + CliPath = cliPath; // null means "use the default dotnet cli"; resolved in DotNetCliCommandExecutor.BuildStartInfo Arguments = arguments; FilePath = filePath; TargetFrameworkMoniker = tfm; @@ -50,7 +50,7 @@ public DotNetCliCommand(string cliPath, string filePath, string tfm, string argu public DotNetCliCommand WithArguments(string arguments) => new(CliPath, FilePath, TargetFrameworkMoniker, arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, LogOutput); - public DotNetCliCommand WithCliPath(string cliPath) + public DotNetCliCommand WithCliPath(FileInfo? cliPath) => new(cliPath, FilePath, TargetFrameworkMoniker, Arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, LogOutput); [PublicAPI] diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs index 4bced8ca12..b8881b6f28 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommandExecutor.cs @@ -134,14 +134,14 @@ internal static void LogEnvVars(DotNetCliCommand command) } } - internal static ProcessStartInfo BuildStartInfo(string? customDotNetCliPath, string workingDirectory, string arguments, + internal static ProcessStartInfo BuildStartInfo(FileInfo? customDotNetCliPath, string workingDirectory, string? arguments, IReadOnlyList? environmentVariables = null, bool redirectStandardInput = false, bool redirectStandardError = true, bool redirectStandardOutput = true) { const string dotnetMultiLevelLookupEnvVarName = "DOTNET_MULTILEVEL_LOOKUP"; var startInfo = new ProcessStartInfo { - FileName = customDotNetCliPath.IsBlank() ? DefaultDotNetCliPath.Value : customDotNetCliPath, + FileName = customDotNetCliPath?.FullName ?? DefaultDotNetCliPath.Value, WorkingDirectory = workingDirectory, Arguments = arguments, UseShellExecute = false, @@ -165,7 +165,7 @@ internal static ProcessStartInfo BuildStartInfo(string? customDotNetCliPath, str foreach (var environmentVariable in environmentVariables) startInfo.Environment[environmentVariable.Key] = environmentVariable.Value; - if (customDotNetCliPath.IsNotBlank() && (environmentVariables == null || environmentVariables.All(envVar => envVar.Key != dotnetMultiLevelLookupEnvVarName))) + if (customDotNetCliPath is not null && (environmentVariables == null || environmentVariables.All(envVar => envVar.Key != dotnetMultiLevelLookupEnvVarName))) startInfo.Environment[dotnetMultiLevelLookupEnvVarName] = "0"; return startInfo; @@ -189,7 +189,7 @@ private static string GetDefaultDotNetCliPath() return "dotnet"; } - internal static async Task GetSdkPathAsync(string cliPath, CancellationToken cancellationToken) + internal static async Task GetSdkPathAsync(FileInfo? cliPath, CancellationToken cancellationToken) { DotNetCliCommand cliCommand = new( cliPath: cliPath, diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs index e77f93be8f..eab089454f 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliExecutor.cs @@ -14,7 +14,7 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli { [PublicAPI] - public class DotNetCliExecutor(string customDotNetCliPath) : IExecutor + public class DotNetCliExecutor(FileInfo? customDotNetCliPath) : IExecutor { public async ValueTask ExecuteAsync(ExecuteParameters executeParameters, CancellationToken cancellationToken) { diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs index 929f9aa272..0f0bb915a2 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliGenerator.cs @@ -1,34 +1,22 @@ -using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; -using JetBrains.Annotations; using System.Diagnostics.CodeAnalysis; using System.Text; namespace BenchmarkDotNet.Toolchains.DotNetCli { - [PublicAPI] - public abstract class DotNetCliGenerator : GeneratorBase + public abstract class DotNetCliGenerator(DotNetCliSettings settings, bool isNetCore) : GeneratorBase { private static readonly string[] ProjectExtensions = [".csproj", ".fsproj", ".vbroj"]; private static readonly string[] SolutionExtensions = [".sln", ".slnx"]; - [PublicAPI] public string TargetFrameworkMoniker { get; } - - [PublicAPI] public string CliPath { get; } - - [PublicAPI] public string PackagesPath { get; } - - protected bool IsNetCore { get; } + /// + /// The settings the toolchain built this generator with. The target framework moniker is already resolved + /// against the runtime (never blank). + /// + public DotNetCliSettings Settings => settings; - [PublicAPI] - protected DotNetCliGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, bool isNetCore) - { - TargetFrameworkMoniker = targetFrameworkMoniker; - CliPath = cliPath.IsBlank() ? "dotnet" : cliPath; - PackagesPath = packagesPath.EnsureNotNull(); - IsNetCore = isNetCore; - } + protected bool IsNetCore => isNetCore; protected override string GetExecutableExtension() => IsNetCore ? ".dll" : ".exe"; @@ -97,13 +85,14 @@ protected override void CopyAllRequiredFiles(ArtifactsPaths artifactsPaths) } } - protected override string GetPackagesDirectoryPath(string buildArtifactsDirectoryPath) => PackagesPath; + protected override string GetPackagesDirectoryPath(string buildArtifactsDirectoryPath) => Settings.PackagesPath?.FullName ?? ""; protected override ValueTask GenerateBuildScriptAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) { + string cli = Settings.CliPath?.FullName ?? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value; var content = new StringBuilder(300) - .AppendLine($"call {CliPath} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath)}") - .AppendLine($"call {CliPath} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, TargetFrameworkMoniker)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetBuildCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, Settings.TargetFrameworkMoniker)}") .ToString(); return new(File.WriteAllTextAsync(artifactsPaths.BuildScriptFilePath, content, cancellationToken)); @@ -119,4 +108,4 @@ private static bool IsRootProjectFolder(DirectoryInfo directoryInfo) .GetFileSystemInfos() .Any(fileInfo => ProjectExtensions.Contains(fileInfo.Extension)); } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs index 66571c7a30..f12b67eaf0 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliPublisher.cs @@ -1,4 +1,3 @@ -using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; @@ -6,27 +5,18 @@ namespace BenchmarkDotNet.Toolchains.DotNetCli; -public class DotNetCliPublisher : IBuilder +public class DotNetCliPublisher( + DotNetCliSettings settings, + string? extraArguments = null, + IReadOnlyList? environmentVariables = null, + bool logOutput = false) : IBuilder { - public string TargetFrameworkMoniker { get; } - public string CustomDotNetCliPath { get; } - public string ExtraArguments { get; } - public IReadOnlyList EnvironmentVariables { get; } - public bool LogOutput { get; } - - public DotNetCliPublisher( - string tfm, - string customDotNetCliPath = "", - string extraArguments = "", - IReadOnlyList? environmentVariables = null, - bool logOutput = false) - { - TargetFrameworkMoniker = tfm; - CustomDotNetCliPath = customDotNetCliPath.EnsureNotNull(); - ExtraArguments = extraArguments.EnsureNotNull(); - EnvironmentVariables = environmentVariables ?? []; - LogOutput = logOutput; - } + // The toolchain resolves the target framework moniker against the runtime before constructing the publisher. + public string TargetFrameworkMoniker { get; } = settings.TargetFrameworkMoniker; + public FileInfo? CustomDotNetCliPath { get; } = settings.CliPath; + public string? ExtraArguments { get; } = extraArguments; + public IReadOnlyList EnvironmentVariables { get; } = environmentVariables ?? []; + public bool LogOutput { get; } = logOutput; public virtual ValueTask BuildAsync(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger, CancellationToken cancellationToken) => new(new DotNetCliCommand( diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliSettings.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliSettings.cs new file mode 100644 index 0000000000..9088e2dfbe --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliSettings.cs @@ -0,0 +1,46 @@ +using System.Collections.Generic; +using BenchmarkDotNet.ConsoleArguments; + +namespace BenchmarkDotNet.Toolchains.DotNetCli; + +/// +/// Settings for toolchains using dotnet cli. +/// +public abstract record DotNetCliSettings : ISettings +{ + /// The path to the dotnet cli to use. If null, the system dotnet will be used. + public FileInfo? CliPath { get; init; } + /// The directory to restore packages to. + public DirectoryInfo? PackagesPath { get; init; } + /// The target framework moniker to build for. If blank, the toolchain derives it from the runtime. + public string TargetFrameworkMoniker { get; init; } = ""; + + internal DotNetCliSettings(CommandLineOptions options) + { + CliPath = options.CliPath; + PackagesPath = options.RestorePath; + } + + protected DotNetCliSettings() { } + + /// + public virtual void FillSettings(IDictionary settings) + { + settings[nameof(CliPath)] = CliPath?.FullName; + settings[nameof(PackagesPath)] = PackagesPath?.FullName; + settings[nameof(TargetFrameworkMoniker)] = TargetFrameworkMoniker; + } + + // FileInfo/DirectoryInfo compare by reference, so compare the paths by value here to keep the record - and the + // toolchain equality built on top of it (used for job deduplication and build partitioning) - value-based. + // Derived records auto-generate equality that chains into this base implementation. + public virtual bool Equals(DotNetCliSettings? other) + => other is not null + && EqualityContract == other.EqualityContract + && CliPath?.FullName == other.CliPath?.FullName + && PackagesPath?.FullName == other.PackagesPath?.FullName + && TargetFrameworkMoniker == other.TargetFrameworkMoniker; + + public override int GetHashCode() + => HashCode.Combine(CliPath?.FullName, PackagesPath?.FullName, TargetFrameworkMoniker); +} diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs deleted file mode 100644 index 0b3aeeb30f..0000000000 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/NetCoreAppSettings.cs +++ /dev/null @@ -1,135 +0,0 @@ -using BenchmarkDotNet.ConsoleArguments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.DotNetCli -{ - /// - /// custom settings used in the auto-generated project.json / .csproj file - /// - [PublicAPI] - public class NetCoreAppSettings - { - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp20 = new("netcoreapp2.0", ".NET Core 2.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp21 = new("netcoreapp2.1", ".NET Core 2.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp22 = new("netcoreapp2.2", ".NET Core 2.2"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp30 = new("netcoreapp3.0", ".NET Core 3.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp31 = new("netcoreapp3.1", ".NET Core 3.1"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp50 = new("net5.0", ".NET 5.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp60 = new("net6.0", ".NET 6.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp70 = new("net7.0", ".NET 7.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp80 = new("net8.0", ".NET 8.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp90 = new("net9.0", ".NET 9.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp10_0 = new("net10.0", ".NET 10.0"); - [PublicAPI] public static readonly NetCoreAppSettings NetCoreApp11_0 = new("net11.0", ".NET 11.0"); - - /// - /// - /// sample values: net6.0, net8.0 - /// - /// - /// used in the auto-generated .csproj file - /// simply ignored if null or empty - /// - /// - /// display name used for showing the results - /// - /// - /// customize dotnet cli path if default is not desired - /// simply ignored if null or empty - /// - /// the directory to restore packages to - /// path to a custom runtime pack - /// path to Mono AOT compiler - /// Mono AOT compiler moder - /// - [PublicAPI] - public NetCoreAppSettings( - string targetFrameworkMoniker, - string runtimeFrameworkVersion, - string name, - string customDotNetCliPath = "", - string packagesPath = "", - string customRuntimePack = "", - string aotCompilerPath = "", - MonoAotCompilerMode aotCompilerMode = MonoAotCompilerMode.mini - ) - { - TargetFrameworkMoniker = targetFrameworkMoniker; - RuntimeFrameworkVersion = runtimeFrameworkVersion.EnsureNotNull(); - Name = name; - - CustomDotNetCliPath = customDotNetCliPath.EnsureNotNull(); - PackagesPath = packagesPath.EnsureNotNull(); - CustomRuntimePack = customRuntimePack.EnsureNotNull(); - AOTCompilerPath = aotCompilerPath.EnsureNotNull(); - AOTCompilerMode = aotCompilerMode; - } - - /// - /// Internal constructor that accept CommandLineOptions - /// - internal NetCoreAppSettings( - string targetFrameworkMoniker, - string runtimeFrameworkVersion, - string name, - CommandLineOptions options) - : this( - targetFrameworkMoniker, - runtimeFrameworkVersion: runtimeFrameworkVersion, - name: name, - customDotNetCliPath: options.CliPath?.FullName ?? "", - packagesPath: options.RestorePath?.FullName ?? "", - customRuntimePack: options.CustomRuntimePack ?? "", - aotCompilerPath: options.AOTCompilerPath?.ToString() ?? "", - aotCompilerMode: options.AOTCompilerMode) - { - } - - internal NetCoreAppSettings(string targetFrameworkMoniker, string name) - : this(targetFrameworkMoniker, runtimeFrameworkVersion: "", name) - { - } - - /// - /// sample values: net6.0, net8.0 - /// - public string TargetFrameworkMoniker { get; } - - public string RuntimeFrameworkVersion { get; } - - /// - /// display name used for showing the results - /// - public string Name { get; } - - public string CustomDotNetCliPath { get; } - - /// - /// The directory to restore packages to. - /// - public string PackagesPath { get; } - - /// - /// Path to a custom runtime pack. - /// - public string CustomRuntimePack { get; } - - /// - /// Path to the Mono AOT Compiler - /// - public string AOTCompilerPath { get; } - - /// - /// Mono AOT Compiler mode, either 'mini' or 'llvm' - /// - public MonoAotCompilerMode AOTCompilerMode { get; } - - public NetCoreAppSettings WithCustomDotNetCliPath(string customDotNetCliPath, string? displayName = null) - => new NetCoreAppSettings(TargetFrameworkMoniker, RuntimeFrameworkVersion, displayName ?? Name, customDotNetCliPath, PackagesPath); - - public NetCoreAppSettings WithCustomPackagesRestorePath(string packagesPath, string? displayName = null) - => new NetCoreAppSettings(TargetFrameworkMoniker, RuntimeFrameworkVersion, displayName ?? Name, CustomDotNetCliPath, packagesPath); - } -} diff --git a/src/BenchmarkDotNet/Toolchains/Executor.cs b/src/BenchmarkDotNet/Toolchains/Executor.cs index 3a8d038556..4c682ad26c 100644 --- a/src/BenchmarkDotNet/Toolchains/Executor.cs +++ b/src/BenchmarkDotNet/Toolchains/Executor.cs @@ -1,7 +1,6 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; @@ -12,13 +11,14 @@ using JetBrains.Annotations; using System.ComponentModel; using System.Diagnostics; -using System.Text; namespace BenchmarkDotNet.Toolchains { [PublicAPI("Used by some of our Superusers that implement their own Toolchains (e.g. Kestrel team)")] public class Executor : IExecutor { + public static readonly Executor Instance = new(); + public async ValueTask ExecuteAsync(ExecuteParameters executeParameters, CancellationToken cancellationToken) { string exePath = executeParameters.BuildResult.ArtifactsPaths.ExecutablePath; @@ -30,7 +30,7 @@ public async ValueTask ExecuteAsync(ExecuteParameters executePara executeParameters.DiagnoserRunMode, cancellationToken).ConfigureAwait(false); } - private static async ValueTask Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, + private async ValueTask Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, IDiagnoser diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, IResolver resolver, int launchIndex, Diagnosers.RunMode diagnoserRunMode, CancellationToken cancellationToken) { @@ -46,7 +46,7 @@ private static async ValueTask Execute(BenchmarkCase benchmarkCas } } - private static async ValueTask ExecuteCore(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, + private async ValueTask ExecuteCore(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, IDiagnoser diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, IResolver resolver, int launchIndex, Diagnosers.RunMode diagnoserRunMode, CancellationToken cancellationToken) { @@ -94,7 +94,7 @@ private static async ValueTask ExecuteCore(BenchmarkCase benchmar results = broker.Results; prefixedOutput = broker.PrefixedOutput; - if (!process.WaitForExit(milliseconds: (int)ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) + if (!process.WaitForExit(milliseconds: (int) ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) { logger.WriteLineInfo("// The benchmarking process did not quit on time, it's going to get force killed now."); } @@ -112,10 +112,13 @@ private static async ValueTask ExecuteCore(BenchmarkCase benchmar launchIndex); } - private static ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, string args, IResolver resolver) + [PublicAPI] + protected virtual ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, string args, IResolver resolver) { var start = new ProcessStartInfo { + FileName = artifactsPaths.ExecutablePath, + Arguments = args, UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardInput = false, @@ -123,61 +126,8 @@ private static ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, Art CreateNoWindow = true, WorkingDirectory = null // by default it's null }; - start.SetEnvironmentVariables(benchmarkCase, resolver); - - string exePath = artifactsPaths.ExecutablePath; - - var runtime = benchmarkCase.GetRuntime(); - - switch (runtime) - { - case ClrRuntime _: - case CoreRuntime _: - case NativeAotRuntime _: - case R2RRuntime _: - start.FileName = exePath; - start.Arguments = args; - break; - case MonoRuntime mono: - start.FileName = mono.CustomPath.IsNotBlank() ? mono.CustomPath : "mono"; - start.Arguments = GetMonoArguments(benchmarkCase.Job, exePath, args, resolver); - break; - case MonoAotLLVMRuntime _: - start.FileName = exePath; - start.Arguments = args; - start.WorkingDirectory = Path.Combine(artifactsPaths.BinariesDirectoryPath, "publish"); - break; - case CustomRuntime _: - start.FileName = exePath; - start.Arguments = args; - break; - default: - throw new NotSupportedException("Runtime = " + runtime); - } return start; } - - private static string GetMonoArguments(Job job, string exePath, string args, IResolver resolver) - { - var arguments = job.HasValue(InfrastructureMode.ArgumentsCharacteristic) - ? job.ResolveValue(InfrastructureMode.ArgumentsCharacteristic, resolver)!.OfType().ToArray() - : []; - - // from mono --help: "Usage is: mono [options] program [program-options]" - var builder = new StringBuilder(30); - - builder.Append(job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.Llvm ? "--llvm" : "--nollvm"); - - foreach (var argument in arguments) - { - builder.Append($" {argument.TextRepresentation}"); - } - - builder.Append($" \"{exePath}\" "); - builder.Append(args); - - return builder.ToString(); - } } } diff --git a/src/BenchmarkDotNet/Toolchains/Framework/CsProjFrameworkToolchain.cs b/src/BenchmarkDotNet/Toolchains/Framework/CsProjFrameworkToolchain.cs new file mode 100644 index 0000000000..0f088e666c --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Framework/CsProjFrameworkToolchain.cs @@ -0,0 +1,94 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Toolchains.Framework; + +public sealed class CsProjFrameworkToolchain : Toolchain, IHasSettings +{ + public static readonly CsProjFrameworkToolchain Net461 = new(ClrRuntime.Net461, FrameworkSettings.Default); + public static readonly CsProjFrameworkToolchain Net462 = new(ClrRuntime.Net462, FrameworkSettings.Default); + public static readonly CsProjFrameworkToolchain Net47 = new(ClrRuntime.Net47, FrameworkSettings.Default); + public static readonly CsProjFrameworkToolchain Net471 = new(ClrRuntime.Net471, FrameworkSettings.Default); + public static readonly CsProjFrameworkToolchain Net472 = new(ClrRuntime.Net472, FrameworkSettings.Default); + public static readonly CsProjFrameworkToolchain Net48 = new(ClrRuntime.Net48, FrameworkSettings.Default); + public static readonly CsProjFrameworkToolchain Net481 = new(ClrRuntime.Net481, FrameworkSettings.Default); + + private CsProjFrameworkToolchain(ClrRuntime runtime, FrameworkSettings settings) + : this(runtime, settings, Resolve(settings, runtime)) { } + + // The build components receive `resolved` (target framework moniker filled in from the runtime); the original + // `settings` is stored for equality and the settings column so an unset moniker is not surfaced as the runtime's. + private CsProjFrameworkToolchain(ClrRuntime runtime, FrameworkSettings settings, FrameworkSettings resolved) + : base("CsProjFramework", + runtime, + new CsProjGenerator(resolved, isNetCore: false), + new DotNetCliBuilder(resolved), + new Executor()) + => Settings = settings; + + // Fills the target framework moniker in from the runtime only when the user left it unset, avoiding both the + // settings copy and the GetTfm() string otherwise. + private static FrameworkSettings Resolve(FrameworkSettings settings, ClrRuntime runtime) + => settings.TargetFrameworkMoniker.IsNotBlank() ? settings : settings with { TargetFrameworkMoniker = runtime.GetTfm() }; + + /// Returns a toolchain for the given runtime and settings. + public static CsProjFrameworkToolchain From(ClrRuntime runtime, FrameworkSettings settings) + { + if (!settings.Equals(FrameworkSettings.Default)) + return new CsProjFrameworkToolchain(runtime, settings); + + return (runtime.Version.Major, runtime.Version.Minor, runtime.Version.Build) switch + { + (4, 8, 1) => Net481, + (4, 8, _) => Net48, + (4, 7, 2) => Net472, + (4, 7, 1) => Net471, + (4, 7, _) => Net47, + (4, 6, 2) => Net462, + (4, 6, 1) => Net461, + _ => new CsProjFrameworkToolchain(runtime, settings), + }; + } + + internal FrameworkSettings Settings { get; } + + ISettings IHasSettings.Settings => Settings; + + public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) + { + await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) + { + yield return validationError; + } + + if (!OsDetector.IsWindows()) + { + yield return new ValidationError(true, + $"{nameof(CsProjFrameworkToolchain)} is supported only for Windows, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + benchmarkCase); + yield break; + } + else if (DotNetSdkValidator.IsCliPathInvalid(((DotNetCliBuilder) Builder).CustomDotNetCliPath, benchmarkCase, out var invalidCliError)) + { + yield return invalidCliError; + } + + foreach (var validationError in DotNetSdkValidator.ValidateFrameworkSdks(benchmarkCase)) + { + yield return validationError; + } + } + + public override bool Equals(object? obj) + => obj is CsProjFrameworkToolchain other + && Runtime.Equals(other.Runtime) + && Settings.Equals(other.Settings); + + public override int GetHashCode() => HashCode.Combine(Runtime, Settings); +} diff --git a/src/BenchmarkDotNet/Toolchains/Framework/FrameworkSettings.cs b/src/BenchmarkDotNet/Toolchains/Framework/FrameworkSettings.cs new file mode 100644 index 0000000000..2a07a0fdd0 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Framework/FrameworkSettings.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.Framework; + +public sealed record FrameworkSettings : DotNetCliSettings +{ + public static readonly FrameworkSettings Default = new(); + + public FrameworkSettings() { } + + internal FrameworkSettings(CommandLineOptions options) : base(options) { } +} diff --git a/src/BenchmarkDotNet/Toolchains/Framework/RoslynFrameworkToolchain.cs b/src/BenchmarkDotNet/Toolchains/Framework/RoslynFrameworkToolchain.cs new file mode 100644 index 0000000000..c4741862a1 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Framework/RoslynFrameworkToolchain.cs @@ -0,0 +1,16 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Toolchains.Roslyn; + +namespace BenchmarkDotNet.Toolchains.Framework; + +public sealed class RoslynFrameworkToolchain : RoslynToolchain +{ + public static readonly RoslynFrameworkToolchain Default = new(ClrRuntime.GetCurrentVersion()); + + private RoslynFrameworkToolchain(ClrRuntime runtime) + : base("RoslynFramework", runtime, RoslynBuilder.Instance, new Executor()) { } + + /// Returns a toolchain for the given runtime. + public static RoslynFrameworkToolchain From(ClrRuntime runtime) + => runtime.Equals(Default.Runtime) ? Default : new RoslynFrameworkToolchain(runtime); +} diff --git a/src/BenchmarkDotNet/Toolchains/IHasSettings.cs b/src/BenchmarkDotNet/Toolchains/IHasSettings.cs new file mode 100644 index 0000000000..7eabf433e7 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/IHasSettings.cs @@ -0,0 +1,11 @@ +namespace BenchmarkDotNet.Toolchains; + +/// +/// Implemented by toolchains that expose a settings record. Enables the summary table to surface settings +/// that differ between benchmarks using the same toolchain (see ). +/// +public interface IHasSettings +{ + /// The toolchain's settings. + ISettings Settings { get; } +} diff --git a/src/BenchmarkDotNet/Toolchains/ISettings.cs b/src/BenchmarkDotNet/Toolchains/ISettings.cs new file mode 100644 index 0000000000..1bfdd104de --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/ISettings.cs @@ -0,0 +1,20 @@ +using System.Collections.Generic; + +namespace BenchmarkDotNet.Toolchains; + +/// +/// A toolchain settings record that can surface its values in the summary table (see ). +/// +public interface ISettings +{ + /// + /// Adds the settings to surface in the summary table. Keys become column names; values may be null. + /// + /// Values are compared with to decide whether a column varies, and rendered + /// via , so they must be value-comparable — use , primitives, + /// enums or records. Emit a path's rather than a + /// /, which compare by reference. + /// + /// + void FillSettings(IDictionary settings); +} diff --git a/src/BenchmarkDotNet/Toolchains/IToolchain.cs b/src/BenchmarkDotNet/Toolchains/IToolchain.cs index 0283dcbcad..5eaab7daac 100644 --- a/src/BenchmarkDotNet/Toolchains/IToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/IToolchain.cs @@ -1,13 +1,13 @@ using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; -using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains; public interface IToolchain { - [PublicAPI] string Name { get; } + Runtime Runtime { get; } IGenerator Generator { get; } IBuilder Builder { get; } IExecutor Executor { get; } diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitSettings.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitSettings.cs index d1f97c394c..8cab9f134b 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitSettings.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitSettings.cs @@ -1,5 +1,6 @@ namespace BenchmarkDotNet.Toolchains.InProcess.Emit; -public class InProcessEmitSettings : InProcessSettings +public record InProcessEmitSettings : InProcessSettings { -} \ No newline at end of file + public static readonly InProcessEmitSettings Default = new(); +} diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitToolchain.cs b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitToolchain.cs index bf6de07eb5..b25cfb7977 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/Emit/InProcessEmitToolchain.cs @@ -1,26 +1,44 @@ -using JetBrains.Annotations; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Portability; namespace BenchmarkDotNet.Toolchains.InProcess.Emit { /// /// An to run the benchmarks in-process by emitting IL. /// - [PublicAPI] - public class InProcessEmitToolchain : Toolchain + public sealed class InProcessEmitToolchain : Toolchain { - /// A toolchain instance with default settings. - public static readonly IToolchain Default = new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = true }); + public static readonly InProcessEmitToolchain Default = new(RuntimeInformation.GetCurrentRuntime(), InProcessEmitSettings.Default); - /// Initializes a new instance of the class. - /// The settings to use for the toolchain. - public InProcessEmitToolchain(InProcessEmitSettings settings) : base( + private readonly InProcessEmitSettings settings; + + public override bool IsInProcess => true; + + private InProcessEmitToolchain(Runtime runtime, InProcessEmitSettings settings) : base( nameof(InProcessEmitToolchain), + runtime, new InProcessEmitGenerator(), new InProcessEmitBuilder(), new InProcessEmitExecutor(settings.ExecuteOnSeparateThread)) { + this.settings = settings; } - public override bool IsInProcess => true; + /// Returns an in-process toolchain for the given settings, associated with the current runtime. + public static InProcessEmitToolchain From(InProcessEmitSettings settings) + => From(RuntimeInformation.GetCurrentRuntime(), settings); + + /// Returns an in-process toolchain for the given runtime and settings. + public static InProcessEmitToolchain From(Runtime runtime, InProcessEmitSettings settings) + => runtime.Equals(Default.Runtime) && settings.Equals(InProcessEmitSettings.Default) + ? Default + : new(runtime, settings); + + public override bool Equals(object? obj) + => obj is InProcessEmitToolchain other + && Runtime.Equals(other.Runtime) + && settings.Equals(other.settings); + + public override int GetHashCode() => HashCode.Combine(Runtime, settings); } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessSettings.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessSettings.cs index b2f3795917..63f09b2d5c 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessSettings.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessSettings.cs @@ -1,6 +1,6 @@ namespace BenchmarkDotNet.Toolchains.InProcess; -public abstract class InProcessSettings +public abstract record InProcessSettings { - public bool ExecuteOnSeparateThread { get; set; } = true; -} \ No newline at end of file + public bool ExecuteOnSeparateThread { get; init; } = true; +} diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs index d7a2da9797..d1ddb12aab 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/InProcessValidator.cs @@ -26,7 +26,6 @@ public class InProcessValidator : IValidator { EnvironmentMode.AffinityCharacteristic, DontValidate }, { EnvironmentMode.JitCharacteristic, ValidateEnvironment }, { EnvironmentMode.PlatformCharacteristic, ValidatePlatform }, - { EnvironmentMode.RuntimeCharacteristic, ValidateEnvironment }, { GcMode.ServerCharacteristic, ValidateEnvironment }, { GcMode.ConcurrentCharacteristic, ValidateEnvironment }, { GcMode.CpuGroupsCharacteristic, ValidateEnvironment }, diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitSettings.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitSettings.cs index dc7bceaaba..a6804fed64 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitSettings.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitSettings.cs @@ -1,6 +1,8 @@ namespace BenchmarkDotNet.Toolchains.InProcess.NoEmit; -public class InProcessNoEmitSettings : InProcessSettings +public record InProcessNoEmitSettings : InProcessSettings { - public IBenchmarkActionFactory? BenchmarkActionFactory { get; set; } -} \ No newline at end of file + public static readonly InProcessNoEmitSettings Default = new(); + + public IBenchmarkActionFactory? BenchmarkActionFactory { get; init; } +} diff --git a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitToolchain.cs b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitToolchain.cs index 7b0c3309a2..cf326f10c6 100644 --- a/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/InProcess/NoEmit/InProcessNoEmitToolchain.cs @@ -1,6 +1,8 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; using JetBrains.Annotations; @@ -13,18 +15,29 @@ namespace BenchmarkDotNet.Toolchains.InProcess.NoEmit; [PublicAPI] public sealed class InProcessNoEmitToolchain : IToolchain { - /// A toolchain instance with default settings. - public static readonly IToolchain Default = new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = true }); + public static readonly InProcessNoEmitToolchain Default = new(RuntimeInformation.GetCurrentRuntime(), InProcessNoEmitSettings.Default); - /// Initializes a new instance of the class. - /// The settings to use for the toolchain. - public InProcessNoEmitToolchain(InProcessNoEmitSettings settings) + private readonly InProcessNoEmitSettings settings; + + private InProcessNoEmitToolchain(Runtime runtime, InProcessNoEmitSettings settings) { + this.settings = settings; + Runtime = runtime; Generator = new InProcessNoEmitGenerator(); Builder = new InProcessNoEmitBuilder(); Executor = new InProcessNoEmitExecutor(settings.ExecuteOnSeparateThread, settings.BenchmarkActionFactory); } + /// Returns an in-process toolchain for the given settings, associated with the current runtime. + public static InProcessNoEmitToolchain From(InProcessNoEmitSettings settings) + => From(RuntimeInformation.GetCurrentRuntime(), settings); + + /// Returns an in-process toolchain for the given runtime and settings. + public static InProcessNoEmitToolchain From(Runtime runtime, InProcessNoEmitSettings settings) + => runtime.Equals(Default.Runtime) && settings.Equals(InProcessNoEmitSettings.Default) + ? Default + : new(runtime, settings); + public async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) { await foreach (var error in InProcessValidator.ValidateAsync(benchmarkCase).ConfigureAwait(false)) @@ -40,25 +53,22 @@ public async IAsyncEnumerable ValidateAsync(BenchmarkCase bench } } - /// Name of the toolchain. - /// The name of the toolchain. - public string Name => nameof(InProcessNoEmitToolchain); + public Runtime Runtime { get; } - /// The generator. - /// The generator. public IGenerator Generator { get; } - /// The builder. - /// The builder. public IBuilder Builder { get; } - /// The executor. - /// The executor. public IExecutor Executor { get; } public bool IsInProcess => true; - /// Returns a that represents this instance. - /// A that represents this instance. public override string ToString() => GetType().Name; -} \ No newline at end of file + + public override bool Equals(object? obj) + => obj is InProcessNoEmitToolchain other + && Runtime.Equals(other.Runtime) + && settings.Equals(other.settings); + + public override int GetHashCode() => HashCode.Combine(Runtime, settings); +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/CsProjMonoCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/CsProjMonoCoreToolchain.cs new file mode 100644 index 0000000000..f010101f03 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/CsProjMonoCoreToolchain.cs @@ -0,0 +1,52 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.Mono; + +public sealed class CsProjMonoCoreToolchain : CsProjNetToolchain +{ + public static readonly CsProjMonoCoreToolchain Mono60 = new(MonoCoreRuntime.Net60, MonoCoreSettings.Default); + public static readonly CsProjMonoCoreToolchain Mono70 = new(MonoCoreRuntime.Net70, MonoCoreSettings.Default); + public static readonly CsProjMonoCoreToolchain Mono80 = new(MonoCoreRuntime.Net80, MonoCoreSettings.Default); + public static readonly CsProjMonoCoreToolchain Mono90 = new(MonoCoreRuntime.Net90, MonoCoreSettings.Default); + public static readonly CsProjMonoCoreToolchain Mono10_0 = new(MonoCoreRuntime.Net10_0, MonoCoreSettings.Default); + public static readonly CsProjMonoCoreToolchain Mono11_0 = new(MonoCoreRuntime.Net11_0, MonoCoreSettings.Default); + + private CsProjMonoCoreToolchain(MonoCoreRuntime runtime, MonoCoreSettings settings) + : this(runtime, settings, Resolve(settings, runtime)) { } + + // The build components receive `resolved` (target framework moniker filled in from the runtime); the original + // `settings` is stored for equality and the settings column so an unset moniker is not surfaced as the runtime's. + private CsProjMonoCoreToolchain(MonoCoreRuntime runtime, MonoCoreSettings settings, MonoCoreSettings resolved) + : base("Mono", runtime, settings, + new CsProjMonoGenerator(resolved), + new MonoPublisher(resolved), + new DotNetCliExecutor(settings.CliPath)) + { + } + + // Fills the target framework moniker in from the runtime only when the user left it unset, avoiding both the + // settings copy and the GetTfm() string otherwise. + private static MonoCoreSettings Resolve(MonoCoreSettings settings, MonoCoreRuntime runtime) + => settings.TargetFrameworkMoniker.IsNotBlank() ? settings : settings with { TargetFrameworkMoniker = runtime.GetTfm() }; + + /// Returns a toolchain for the given runtime and settings. + public static CsProjMonoCoreToolchain From(MonoCoreRuntime runtime, MonoCoreSettings settings) + { + if (!settings.Equals(MonoCoreSettings.Default)) + return new CsProjMonoCoreToolchain(runtime, settings); + + return runtime.Version.Major switch + { + 6 => Mono60, + 7 => Mono70, + 8 => Mono80, + 9 => Mono90, + 10 => Mono10_0, + 11 => Mono11_0, + _ => new CsProjMonoCoreToolchain(runtime, settings), + }; + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs b/src/BenchmarkDotNet/Toolchains/Mono/CsProjMonoGenerator.cs similarity index 62% rename from src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs rename to src/BenchmarkDotNet/Toolchains/Mono/CsProjMonoGenerator.cs index 543283bd0c..fb8c1bf2d8 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/CsProjMonoGenerator.cs @@ -4,21 +4,19 @@ namespace BenchmarkDotNet.Toolchains.Mono { - public class MonoGenerator : CsProjGenerator + internal sealed class CsProjMonoGenerator(MonoCoreSettings settings) : CsProjGenerator(settings) { - public MonoGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string runtimeFrameworkVersion) : base(targetFrameworkMoniker, cliPath, packagesPath, runtimeFrameworkVersion, true) - { - } - protected override string GetRuntimeSettings(GcMode gcMode, IResolver resolver) { // Workaround for following issues. // 1. 'Found multiple publish output files with the same relative path' error - // 2. NU1102 error occurs when passing /p:UseMonoRuntime=true to the dotnet cli with projects containing .NET 9.0 or higher. #3000 + // 2. NU1102 error occurs when running 'dotnet publish' on projects that contain .NET 9.0 or higher. https://github.com/dotnet/BenchmarkDotNet/issues/3000 return base.GetRuntimeSettings(gcMode, resolver) + """ false + + true """; diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotBuilder.cs b/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoAotBuilder.cs similarity index 64% rename from src/BenchmarkDotNet/Toolchains/Mono/MonoAotBuilder.cs rename to src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoAotBuilder.cs index 1e8142a4af..72b938c8ad 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotBuilder.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoAotBuilder.cs @@ -1,33 +1,27 @@ -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.Results; -using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.Mono { - [PublicAPI] - public class MonoAotBuilder : IBuilder + internal sealed class LegacyMonoAotBuilder(MonoAotSettings settings) : IBuilder { - [PublicAPI] public async ValueTask BuildAsync(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger, CancellationToken cancellationToken) { - var result = await Roslyn.Builder.Instance.BuildAsync(generateResult, buildPartition, logger, cancellationToken).ConfigureAwait(false); + var result = await Roslyn.RoslynBuilder.Instance.BuildAsync(generateResult, buildPartition, logger, cancellationToken).ConfigureAwait(false); if (!result.IsBuildSuccess) return result; var exePath = generateResult.ArtifactsPaths.ExecutablePath; - var monoRuntime = (MonoRuntime)buildPartition.Runtime; - var environmentVariables = monoRuntime.MonoBclPath.IsBlank() + var environmentVariables = settings.MonoBclPath is null ? null - : new Dictionary { { "MONO_PATH", monoRuntime.MonoBclPath } }; + : new Dictionary { { "MONO_PATH", settings.MonoBclPath.FullName } }; var (exitCode, output) = await ProcessHelper.RunAndReadOutputLineByLineAsync( - fileName: monoRuntime.CustomPath.IsNotBlank() ? monoRuntime.CustomPath : "mono", - arguments: $"{monoRuntime.AotArgs} \"{Path.GetFullPath(exePath)}\"", + fileName: settings.MonoPath?.FullName ?? "mono", + arguments: $"{settings.AotArgs} \"{Path.GetFullPath(exePath)}\"", workingDirectory: Path.GetDirectoryName(exePath)!, environmentVariables: environmentVariables, includeErrors: true, @@ -40,4 +34,4 @@ public async ValueTask BuildAsync(GenerateResult generateResult, Bu : result; } } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoExecutor.cs b/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoExecutor.cs new file mode 100644 index 0000000000..1735675cc1 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoExecutor.cs @@ -0,0 +1,47 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Running; +using System.Diagnostics; +using System.Text; + +namespace BenchmarkDotNet.Toolchains.Mono; + +internal sealed class LegacyMonoExecutor(LegacyMonoSettings settings) : Executor +{ + internal LegacyMonoSettings Settings => settings; + + protected override ProcessStartInfo CreateStartInfo(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, string args, IResolver resolver) + { + var start = base.CreateStartInfo(benchmarkCase, artifactsPaths, args, resolver); + start.FileName = settings.MonoPath?.FullName ?? "mono"; + start.Arguments = GetMonoArguments(benchmarkCase.Job, artifactsPaths.ExecutablePath, args, resolver); + if (settings.MonoBclPath is not null) + { + start.EnvironmentVariables["MONO_PATH"] = settings.MonoBclPath.FullName; + } + return start; + } + + private static string GetMonoArguments(Job job, string exePath, string args, IResolver resolver) + { + var arguments = job.HasValue(InfrastructureMode.ArgumentsCharacteristic) + ? job.ResolveValue(InfrastructureMode.ArgumentsCharacteristic, resolver)!.OfType().ToArray() + : []; + + // from mono --help: "Usage is: mono [options] program [program-options]" + var builder = new StringBuilder(30); + + builder.Append(job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver) == Jit.Llvm ? "--llvm" : "--nollvm"); + + foreach (var argument in arguments) + { + builder.Append($" {argument.TextRepresentation}"); + } + + builder.Append($" \"{exePath}\" "); + builder.Append(args); + + return builder.ToString(); + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoSettings.cs b/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoSettings.cs new file mode 100644 index 0000000000..865b58c354 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/LegacyMonoSettings.cs @@ -0,0 +1,39 @@ +using System.Collections.Generic; +using BenchmarkDotNet.ConsoleArguments; + +namespace BenchmarkDotNet.Toolchains.Mono; + +/// Settings for legacy Mono toolchains. +public abstract record LegacyMonoSettings : ISettings +{ + /// Optional path to the Mono installation. + public FileInfo? MonoPath { get; init; } + /// Optional path to the Mono Base Class Library (BCL) directory. + public DirectoryInfo? MonoBclPath { get; init; } + + protected LegacyMonoSettings() { } + + internal LegacyMonoSettings(CommandLineOptions options) + { + MonoPath = options.MonoPath; + } + + /// + public virtual void FillSettings(IDictionary settings) + { + settings[nameof(MonoPath)] = MonoPath?.FullName; + settings[nameof(MonoBclPath)] = MonoBclPath?.FullName; + } + + // FileInfo/DirectoryInfo compare by reference, so compare the paths by value here to keep the record - and the + // toolchain equality built on top of it (used for job deduplication and build partitioning) - value-based. + // Derived records auto-generate equality that chains into this base implementation. + public virtual bool Equals(LegacyMonoSettings? other) + => other is not null + && EqualityContract == other.EqualityContract + && MonoPath?.FullName == other.MonoPath?.FullName + && MonoBclPath?.FullName == other.MonoBclPath?.FullName; + + public override int GetHashCode() + => HashCode.Combine(MonoPath?.FullName, MonoBclPath?.FullName); +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotSettings.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotSettings.cs new file mode 100644 index 0000000000..db87412919 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotSettings.cs @@ -0,0 +1,26 @@ +using System.Collections.Generic; +using BenchmarkDotNet.ConsoleArguments; + +namespace BenchmarkDotNet.Toolchains.Mono; + +public sealed record MonoAotSettings : LegacyMonoSettings +{ + public static readonly MonoAotSettings Default = new(); + + public MonoAotSettings() { } + + internal MonoAotSettings(CommandLineOptions options) : base(options) { } + + /// + /// Aot args for the build. + /// Example: "--aot=full,llvm". See https://www.mono-project.com/docs/advanced/aot/ for more details. + /// + public string AotArgs { get; init; } = "--aot"; + + /// + public override void FillSettings(IDictionary settings) + { + base.FillSettings(settings); + settings[nameof(AotArgs)] = AotArgs; + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs deleted file mode 100644 index ba9fa1f54a..0000000000 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoAotToolchain.cs +++ /dev/null @@ -1,51 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.Roslyn; -using BenchmarkDotNet.Validators; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.Mono -{ - public class MonoAotToolchain : Toolchain - { - public static readonly IToolchain Instance = new MonoAotToolchain(); - - [PublicAPI] - public MonoAotToolchain() : base("MonoAot", new Generator(), new MonoAotBuilder(), new Executor()) - { - } - - public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) - { - await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) - { - yield return validationError; - } - - if (!benchmarkCase.Job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic) || benchmarkCase.Job.Environment.Runtime is not MonoRuntime) - { - yield return new ValidationError(true, - "The MonoAOT toolchain requires the Runtime property to be configured explicitly to an instance of MonoRuntime class", - benchmarkCase); - } - - if ((benchmarkCase.Job.Environment.Runtime is MonoRuntime monoRuntime) && monoRuntime.MonoBclPath.IsNotBlank() && !Directory.Exists(monoRuntime.MonoBclPath)) - { - yield return new ValidationError(true, - $"The MonoBclPath provided for MonoAOT toolchain: {monoRuntime.MonoBclPath} does NOT exist.", - benchmarkCase); - } - - if (benchmarkCase.Job.HasValue(InfrastructureMode.BuildConfigurationCharacteristic) - && benchmarkCase.Job.ResolveValue(InfrastructureMode.BuildConfigurationCharacteristic, resolver) != InfrastructureMode.ReleaseConfigurationName) - { - yield return new ValidationError(true, - "The MonoAOT toolchain does not allow to rebuild source project, so defining custom build configuration makes no sense", - benchmarkCase); - } - } - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoCoreSettings.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoCoreSettings.cs new file mode 100644 index 0000000000..c9928997a3 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoCoreSettings.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.Mono; + +public sealed record MonoCoreSettings : DotNetCliSettings +{ + public static readonly MonoCoreSettings Default = new(); + + public MonoCoreSettings() { } + + internal MonoCoreSettings(CommandLineOptions options) : base(options) { } +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs index ba5d4c9572..218609691f 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoPublisher.cs @@ -1,11 +1,12 @@ using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.Results; namespace BenchmarkDotNet.Toolchains.Mono; -public class MonoPublisher(string tfm, string customDotNetCliPath) : DotNetCliPublisher(tfm, customDotNetCliPath) +internal sealed class MonoPublisher(DotNetCliSettings settings) : DotNetCliPublisher(settings) { public override async ValueTask BuildAsync(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger, CancellationToken cancellationToken) { @@ -27,7 +28,7 @@ public override async ValueTask BuildAsync(GenerateResult generateR private static string GetExtraArguments() { - var runtimeIdentifier = CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier(); + var runtimeIdentifier = RuntimeInformation.GetPortableRuntimeIdentifier(); // /p:RuntimeIdentifiers is set explicitly here because --self-contained requires it, see https://github.com/dotnet/sdk/issues/10566 return $"--self-contained -r {runtimeIdentifier} /p:RuntimeIdentifiers={runtimeIdentifier}"; } diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs deleted file mode 100644 index cc1a7db0f7..0000000000 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoToolchain.cs +++ /dev/null @@ -1,44 +0,0 @@ -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.Mono -{ - [PublicAPI] - public class MonoToolchain : CsProjCoreToolchain, IEquatable - { - [PublicAPI] public static readonly IToolchain Mono60 = From(new NetCoreAppSettings("net6.0", "mono60")); - [PublicAPI] public static readonly IToolchain Mono70 = From(new NetCoreAppSettings("net7.0", "mono70")); - [PublicAPI] public static readonly IToolchain Mono80 = From(new NetCoreAppSettings("net8.0", "mono80")); - [PublicAPI] public static readonly IToolchain Mono90 = From(new NetCoreAppSettings("net9.0", "mono90")); - [PublicAPI] public static readonly IToolchain Mono10_0 = From(new NetCoreAppSettings("net10.0", "mono10_0")); - [PublicAPI] public static readonly IToolchain Mono11_0 = From(new NetCoreAppSettings("net11.0", "mono11_0")); - - private MonoToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) - : base(name, generator, builder, executor, customDotNetCliPath) - { - } - - [PublicAPI] - public static new IToolchain From(NetCoreAppSettings settings) - => new MonoToolchain(settings.Name, - new MonoGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, settings.RuntimeFrameworkVersion), - new MonoPublisher(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath), - new DotNetCliExecutor(settings.CustomDotNetCliPath), - settings.CustomDotNetCliPath); - - public override bool Equals(object? obj) => obj is MonoToolchain typed && Equals(typed); - - public bool Equals(MonoToolchain? other) - { - if (ReferenceEquals(null, other)) - return false; - if (ReferenceEquals(this, other)) - return true; - - return Generator.Equals(other.Generator); - } - - public override int GetHashCode() => Generator.GetHashCode(); - } -} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoAotToolchain.cs new file mode 100644 index 0000000000..11bfc156bc --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoAotToolchain.cs @@ -0,0 +1,53 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Toolchains.Mono; + +public sealed class RoslynMonoAotToolchain : RoslynToolchain, IHasSettings +{ + public static readonly RoslynMonoAotToolchain Default = new(MonoAotSettings.Default); + + private RoslynMonoAotToolchain(MonoAotSettings settings) + : base("RoslynMonoAot", MonoAotRuntime.Default, new LegacyMonoAotBuilder(settings), new LegacyMonoExecutor(settings)) + => Settings = settings; + + /// Returns a toolchain for the given settings. + public static RoslynMonoAotToolchain From(MonoAotSettings settings) + => settings.Equals(MonoAotSettings.Default) ? Default : new RoslynMonoAotToolchain(settings); + + internal MonoAotSettings Settings { get; } + + ISettings IHasSettings.Settings => Settings; + + public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) + { + await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) + { + yield return validationError; + } + + if (Settings.MonoBclPath is { Exists: false }) + { + yield return new ValidationError(true, + $"MonoBclPath provided for {nameof(RoslynMonoAotToolchain)}: \"{Settings.MonoBclPath}\" does NOT exist.", + benchmarkCase); + } + + if (Settings.MonoPath is null && !HostEnvironmentInfo.GetCurrent().IsMonoInstalled.Value) + { + yield return new ValidationError(true, + $"Mono is not installed or added to PATH", + benchmarkCase); + } + } + + public override bool Equals(object? obj) + => obj is RoslynMonoAotToolchain other + && Runtime.Equals(other.Runtime) + && Settings.Equals(other.Settings); + + public override int GetHashCode() => HashCode.Combine(Runtime, Settings); +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoSettings.cs b/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoSettings.cs new file mode 100644 index 0000000000..a70060704a --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoSettings.cs @@ -0,0 +1,12 @@ +using BenchmarkDotNet.ConsoleArguments; + +namespace BenchmarkDotNet.Toolchains.Mono; + +public sealed record MonoSettings : LegacyMonoSettings +{ + public static readonly MonoSettings Default = new(); + + public MonoSettings() { } + + internal MonoSettings(CommandLineOptions options) : base(options) { } +} diff --git a/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoToolchain.cs b/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoToolchain.cs new file mode 100644 index 0000000000..50dd74e0ff --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Mono/RoslynMonoToolchain.cs @@ -0,0 +1,53 @@ +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Toolchains.Mono; + +public sealed class RoslynMonoToolchain : RoslynToolchain, IHasSettings +{ + public static readonly RoslynMonoToolchain Default = new(MonoSettings.Default); + + private RoslynMonoToolchain(MonoSettings settings) + : base("RoslynMono", MonoRuntime.Default, RoslynBuilder.Instance, new LegacyMonoExecutor(settings)) + => Settings = settings; + + /// Returns a toolchain for the given settings. + public static RoslynMonoToolchain From(MonoSettings settings) + => settings.Equals(MonoSettings.Default) ? Default : new RoslynMonoToolchain(settings); + + internal MonoSettings Settings { get; } + + ISettings IHasSettings.Settings => Settings; + + public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) + { + await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) + { + yield return validationError; + } + + if (Settings.MonoBclPath is { Exists: false }) + { + yield return new ValidationError(true, + $"MonoBclPath provided for {nameof(RoslynMonoToolchain)}: \"{Settings.MonoBclPath}\" does NOT exist.", + benchmarkCase); + } + + if (Settings.MonoPath is null && !HostEnvironmentInfo.GetCurrent().IsMonoInstalled.Value) + { + yield return new ValidationError(true, + $"Mono is not installed or added to PATH", + benchmarkCase); + } + } + + public override bool Equals(object? obj) + => obj is RoslynMonoToolchain other + && Runtime.Equals(other.Runtime) + && Settings.Equals(other.Settings); + + public override int GetHashCode() => HashCode.Combine(Runtime, Settings); +} diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotCompilerMode.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotCompilerMode.cs deleted file mode 100644 index 2ea48ec37d..0000000000 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotCompilerMode.cs +++ /dev/null @@ -1,9 +0,0 @@ -namespace BenchmarkDotNet.Toolchains.MonoAotLLVM -{ - public enum MonoAotCompilerMode - { - mini = 0, // default - llvm, - wasm - } -} diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs deleted file mode 100644 index 0253e4fc85..0000000000 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ /dev/null @@ -1,69 +0,0 @@ -using BenchmarkDotNet.Detectors; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Helpers; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using System.Text; -using System.Xml; - -namespace BenchmarkDotNet.Toolchains.MonoAotLLVM -{ - public class MonoAotLLVMGenerator : CsProjGenerator - { - private readonly string CustomRuntimePack; - private readonly string AotCompilerPath; - private readonly MonoAotCompilerMode AotCompilerMode; - - public MonoAotLLVMGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string customRuntimePack, string aotCompilerPath, MonoAotCompilerMode aotCompilerMode) - : base(targetFrameworkMoniker, cliPath, packagesPath) - { - CustomRuntimePack = customRuntimePack; - AotCompilerPath = aotCompilerPath; - AotCompilerMode = aotCompilerMode; - BenchmarkRunCallType = Code.CodeGenBenchmarkRunCallType.Direct; - } - - protected override async ValueTask GenerateProjectAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) - { - BenchmarkCase benchmark = buildPartition.RepresentativeBenchmarkCase; - var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); - - string useLLVM = AotCompilerMode == MonoAotCompilerMode.llvm ? "true" : "false"; - - var xmlDoc = new XmlDocument(); - xmlDoc.Load(projectFile.FullName); - var (customProperties, sdkName) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); - - string content = new StringBuilder(await ResourceHelper.LoadTemplateAsync("MonoAOTLLVMCsProj.txt", cancellationToken).ConfigureAwait(false)) - .Replace("$PLATFORM$", buildPartition.Platform.ToConfig()) - .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) - .Replace("$CSPROJPATH$", projectFile.FullName) - .Replace("$TFM$", TargetFrameworkMoniker) - .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) - .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$SDKNAME$", sdkName) - .Replace("$RUNTIMEPACK$", CustomRuntimePack) - .Replace("$COMPILERBINARYPATH$", AotCompilerPath) - .Replace("$RUNTIMEIDENTIFIER$", CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier()) - .Replace("$USELLVM$", useLLVM) - .ToString(); - - await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); - - await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); - } - - protected override string GetPublishDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(GetBinariesDirectoryPath(buildArtifactsDirectoryPath, configuration), "publish"); - - protected override string GetExecutablePath(string binariesDirectoryPath, string programName) - => OsDetector.IsWindows() - ? Path.Combine(binariesDirectoryPath, "publish", $"{programName}.exe") - : Path.Combine(binariesDirectoryPath, "publish", programName); - - protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier()); - } -} diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs deleted file mode 100644 index 7ba5c923a2..0000000000 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMToolChain.cs +++ /dev/null @@ -1,44 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Validators; - -namespace BenchmarkDotNet.Toolchains.MonoAotLLVM -{ - public class MonoAotLLVMToolChain : Toolchain - { - private readonly string _customDotNetCliPath; - - public MonoAotLLVMToolChain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) - : base(name, generator, builder, executor) - { - _customDotNetCliPath = customDotNetCliPath; - } - - public static IToolchain From(NetCoreAppSettings netCoreAppSettings) - => new MonoAotLLVMToolChain(netCoreAppSettings.Name, - new MonoAotLLVMGenerator(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath, - netCoreAppSettings.PackagesPath, - netCoreAppSettings.CustomRuntimePack, - netCoreAppSettings.AOTCompilerPath, - netCoreAppSettings.AOTCompilerMode), - new DotNetCliBuilder(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath), - new Executor(), - netCoreAppSettings.CustomDotNetCliPath); - - public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) - { - await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) - { - yield return validationError; - } - - foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(_customDotNetCliPath, benchmarkCase)) - { - yield return validationError; - } - } - } -} diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs deleted file mode 100644 index 15cd4c6fe2..0000000000 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmToolchain.cs +++ /dev/null @@ -1,48 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Validators; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.MonoWasm -{ - [PublicAPI] - public class WasmToolchain : Toolchain - { - private string CustomDotNetCliPath { get; } - - private WasmToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) - : base(name, generator, builder, executor) - { - CustomDotNetCliPath = customDotNetCliPath; - } - - public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) - { - await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) - { - yield return validationError; - } - - foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) - { - yield return validationError; - } - } - - [PublicAPI] - public static IToolchain From(NetCoreAppSettings netCoreAppSettings) - => new WasmToolchain(netCoreAppSettings.Name, - new WasmGenerator(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath, - netCoreAppSettings.PackagesPath, - netCoreAppSettings.CustomRuntimePack, - netCoreAppSettings.AOTCompilerMode == MonoAotLLVM.MonoAotCompilerMode.wasm), - new DotNetCliPublisher(netCoreAppSettings.TargetFrameworkMoniker, - netCoreAppSettings.CustomDotNetCliPath, - // aot builds can be very slow - logOutput: netCoreAppSettings.AOTCompilerMode == MonoAotLLVM.MonoAotCompilerMode.wasm), - new WasmExecutor(), - netCoreAppSettings.CustomDotNetCliPath); - } -} \ No newline at end of file diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/CsProjNativeAotGenerator.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/CsProjNativeAotGenerator.cs new file mode 100644 index 0000000000..07dfd191eb --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/CsProjNativeAotGenerator.cs @@ -0,0 +1,329 @@ +using BenchmarkDotNet.Detectors; +using BenchmarkDotNet.Detectors.Cpu; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using System.Text; +using System.Xml; + +namespace BenchmarkDotNet.Toolchains.NativeAot; + +/// +/// Generates new csproj file for self-contained NativeAOT app. +/// +internal sealed class CsProjNativeAotGenerator : CsProjGenerator +{ + internal const string NativeAotNuGetFeed = "nativeAotNuGetFeed"; + private const string DefaultNuGetFeed = "https://api.nuget.org/v3/index.json"; + private const string LocalBuildDotNetFeed = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet11/nuget/v3/index.json"; + internal const string GeneratedRdXmlFileName = "bdn_generated.rd.xml"; + + private readonly NativeAotSettings settings; + + internal CsProjNativeAotGenerator(NativeAotSettings settings) + : base(settings with { PackagesPath = GetPackagesDirectoryPath(settings.UseTempFolderForRestore, settings.PackagesPath) }) + { + this.settings = settings; + BenchmarkRunCallType = Code.CodeGenBenchmarkRunCallType.Direct; + } + + protected override string GetExecutableExtension() => OsDetector.ExecutableExtension; + + protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) + => settings.UseTempFolderForRestore + ? Path.Combine(Path.GetTempPath(), programName) // store everything in temp to avoid collisions with IDE + : base.GetBuildArtifactsDirectoryPath(buildPartition, programName); + + protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, settings.RuntimeIdentifier, "publish"); + + protected override ValueTask GenerateBuildScriptAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) + { + string projectFilePath = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, NullLogger.Instance).FullName; + string extraArguments = CsProjNativeAotToolchain.GetExtraArguments(settings.RuntimeIdentifier); + + string cli = Settings.CliPath?.FullName ?? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value; + var content = new StringBuilder(300) + .AppendLine($"call {cli} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, projectFilePath, extraArguments)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetPublishCommand(artifactsPaths, buildPartition, projectFilePath, Settings.TargetFrameworkMoniker, extraArguments)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, extraArguments)}") + .AppendLine($"call {cli} {DotNetCliCommand.GetPublishCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, Settings.TargetFrameworkMoniker, extraArguments)}") + .ToString(); + + return new(File.WriteAllTextAsync(artifactsPaths.BuildScriptFilePath, content, cancellationToken)); + } + + // We always want to have a new directory for NuGet packages restore. + // Some of the packages are going to contain source code, so they can not be in the subfolder of current solution + // otherwise they would be compiled too (new .csproj include all .cs files from subfolders by default). + private static DirectoryInfo? GetPackagesDirectoryPath(bool useTempFolderForRestore, DirectoryInfo? packagesRestorePath) + => packagesRestorePath is null && useTempFolderForRestore + ? new DirectoryInfo(Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString())) + : null; + + protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) + => settings.UseTempFolderForRestore && artifactsPaths.PackagesDirectoryName.IsNotBlank() + ? base.GetArtifactsToCleanup(artifactsPaths).Concat([artifactsPaths.PackagesDirectoryName]).ToArray() + : base.GetArtifactsToCleanup(artifactsPaths); + + protected override async ValueTask GenerateNuGetConfigAsync(ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) + { + var feeds = GetFeeds(); + if (feeds.Length == 0) + return; + + // Skip creating a NuGet.config if the clear tag is not specified and the only feed is the default nuget.org feed. + if (!settings.UseNuGetClearTag && feeds is [{ Value: DefaultNuGetFeed }]) + return; + + string content = $""" + + + + {(settings.UseNuGetClearTag ? "" : string.Empty)} + {string.Join(Environment.NewLine + " ", feeds.Select(feed => $""))} + + + """; + + await File.WriteAllTextAsync(artifactsPaths.NuGetConfigPath, content, cancellationToken).ConfigureAwait(false); + } + + // The ILCompiler is restored either from a NuGet feed, or from a local runtime build (which also needs the dotnet nightly feed). + private KeyValuePair[] GetFeeds() + => settings.LocalIlcPackages is not null ? [new("local", settings.LocalIlcPackages.FullName), new("dotnet11", LocalBuildDotNetFeed)] + : settings.NuGetFeedUrl.IsNotBlank() ? [new(NativeAotNuGetFeed, settings.NuGetFeedUrl!)] + : []; + + protected override async ValueTask GenerateProjectAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) + { + var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).FullName; + + await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, GenerateProjectForNuGetBuild(projectFile, buildPartition, artifactsPaths, logger), cancellationToken).ConfigureAwait(false); + + await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); + await GenerateReflectionFileAsync(artifactsPaths, cancellationToken).ConfigureAwait(false); + } + + private string GenerateProjectForNuGetBuild(string projectFilePath, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) => $""" + + + Exe + {Settings.TargetFrameworkMoniker} + {settings.RuntimeIdentifier} + {artifactsPaths.ProgramName} + {artifactsPaths.ProgramName} + true + {buildPartition.Platform.ToConfig()} + False + false + false + false + true + {settings.OptimizationPreference} + {settings.OptimizationPreference} + {settings.GenerateStackTraceData} + {settings.GenerateStackTraceData} + false + false + false + {GetInstructionSetSettings(buildPartition)} + + {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)} + + + + + {GetILCompilerPackageReference()} + + + + {string.Join(Environment.NewLine, GetRdXmlFiles(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).Select(file => $""))} + + {GetCustomProperties(buildPartition, logger)} + + + latest + + + """; + + private string GetCustomProperties(BuildPartition buildPartition, ILogger logger) + { + var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); + var xmlDoc = new XmlDocument(); + xmlDoc.Load(projectFile.FullName); + + (string customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); + return customProperties; + } + + + private string GetILCompilerPackageReference() + => settings.IlCompilerVersion.IsBlank() ? "" : $@""; + + private string GetInstructionSetSettings(BuildPartition buildPartition) + { + string instructionSet = settings.InstructionSet.IsBlank() + ? GetCurrentInstructionSet(buildPartition.Platform) + : settings.InstructionSet; + + return instructionSet.IsNotBlank() + ? $"{instructionSet}" + : ""; + } + + public IEnumerable GetRdXmlFiles(Type benchmarkTarget, ILogger logger) + { + yield return GeneratedRdXmlFileName; + + var projectFile = GetProjectFilePath(benchmarkTarget, logger); + var projectFileFolder = projectFile.DirectoryName!; + var rdXml = Path.Combine(projectFileFolder, "rd.xml"); + if (File.Exists(rdXml)) + { + yield return rdXml; + } + + foreach (var item in Directory.GetFiles(projectFileFolder, "*.rd.xml")) + { + yield return item; + } + } + + /// + /// mandatory to make it possible to call GC.GetAllocatedBytesForCurrentThread() using reflection (not part of .NET Standard) + /// + private ValueTask GenerateReflectionFileAsync(ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) + { + const string content = """ + + + + + + + + + + + + + + + """; + + string directoryName = Path.GetDirectoryName(artifactsPaths.ProjectFilePath)!; + if (directoryName == null) + throw new InvalidOperationException($"Can't get directory of projectFilePath ('{artifactsPaths.ProjectFilePath}')"); + + return new(File.WriteAllTextAsync(Path.Combine(directoryName, GeneratedRdXmlFileName), content, cancellationToken)); + } + + private string GetCurrentInstructionSet(Platform platform) + => string.Join(",", GetCurrentProcessInstructionSets(platform)); + + // based on https://github.com/dotnet/runtime/tree/v10.0.0-rc.1.25451.107/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt + private IEnumerable GetCurrentProcessInstructionSets(Platform platform) + { + if (!Runtime.TryParse(Settings.TargetFrameworkMoniker, out Runtime? runtime)) + { + throw new NotSupportedException($"Invalid TFM: '{Settings.TargetFrameworkMoniker}'"); + } + + // The instruction sets recognized by ILC depend on the .NET version being compiled; gate on the version directly. + Version version = runtime.Version!; + + if (platform == RuntimeInformation.GetCurrentPlatform() // "native" does not support cross-compilation (so does BDN for now) + && version.Major >= 8) + { + yield return "native"; // added in .NET 8 https://github.com/dotnet/runtime/pull/87865 + yield break; + } + + switch (platform) + { + case Platform.X86: + case Platform.X64: + if (HardwareIntrinsics.IsX86BaseSupported) yield return "base"; + if (HardwareIntrinsics.IsX86Sse42Supported) + { + if (version.Major <= 10) yield return "sse4.2"; + if (version.Major <= 9) yield return "popcnt"; + } + if (HardwareIntrinsics.IsX86AvxSupported) yield return "avx"; + if (HardwareIntrinsics.IsX86Avx2Supported) + { + yield return "avx2"; + + if (version.Major <= 9) + { + yield return "bmi"; + yield return "bmi2"; + yield return "fma"; + yield return "lzcnt"; + } + } + if (HardwareIntrinsics.IsX86Avx512Supported && (version.Major > 8)) + { + if (version.Major >= 10) + { + yield return "avx512"; + } + else + { + yield return "avx512f"; + yield return "avx512f_vl"; + yield return "avx512bw"; + yield return "avx512bw_vl"; + yield return "avx512cd"; + yield return "avx512cd_vl"; + yield return "avx512dq"; + yield return "avx512dq_vl"; + } + } + if (HardwareIntrinsics.IsX86Avx512v2Supported && (version.Major > 8)) + { + if (version.Major >= 10) + { + yield return "avx512v2"; + } + else + { + yield return "avx512vbmi"; + yield return "avx512vbmi_vl"; + } + } + if (HardwareIntrinsics.IsX86Avx512v3Supported && (version.Major >= 10)) yield return "avx512v3"; + if (HardwareIntrinsics.IsX86Avx10v1Supported && (version.Major >= 9)) yield return "avx10v1"; + if (HardwareIntrinsics.IsX86Avx10v2Supported && (version.Major >= 10)) yield return "avx10v2"; + if (HardwareIntrinsics.IsX86AesSupported) + { + yield return "aes"; + if (version.Major <= 9) yield return "pclmul"; + } + if (HardwareIntrinsics.IsX86AvxVnniSupported) yield return "avxvnni"; + if (HardwareIntrinsics.IsX86SerializeSupported && version.Major > 7) yield return "serialize"; // https://github.com/dotnet/BenchmarkDotNet/issues/2463#issuecomment-1809625008 + break; + case Platform.Arm64: + if (HardwareIntrinsics.IsArmBaseSupported) + { + yield return "base"; + yield return "neon"; + } + if (HardwareIntrinsics.IsArmAesSupported) yield return "aes"; + if (HardwareIntrinsics.IsArmCrc32Supported) yield return "crc"; + if (HardwareIntrinsics.IsArmDpSupported) yield return "dotprod"; + if (HardwareIntrinsics.IsArmRdmSupported) yield return "rdma"; + if (HardwareIntrinsics.IsArmSha1Supported) yield return "sha1"; + if (HardwareIntrinsics.IsArmSha256Supported) yield return "sha2"; + break; + default: + yield break; + } + } +} diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/CsProjNativeAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/CsProjNativeAotToolchain.cs new file mode 100644 index 0000000000..a412b644f7 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/CsProjNativeAotToolchain.cs @@ -0,0 +1,57 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.NativeAot; + +public sealed class CsProjNativeAotToolchain : CsProjNetToolchain +{ + /// compiled as net7.0 + public static readonly CsProjNativeAotToolchain Net70 = new(NativeAotRuntime.Net70, NativeAotSettings.Default); + /// compiled as net8.0 + public static readonly CsProjNativeAotToolchain Net80 = new(NativeAotRuntime.Net80, NativeAotSettings.Default); + /// compiled as net9.0 + public static readonly CsProjNativeAotToolchain Net90 = new(NativeAotRuntime.Net90, NativeAotSettings.Default); + /// compiled as net10.0 + public static readonly CsProjNativeAotToolchain Net10_0 = new(NativeAotRuntime.Net10_0, NativeAotSettings.Default); + /// compiled as net11.0 + public static readonly CsProjNativeAotToolchain Net11_0 = new(NativeAotRuntime.Net11_0, NativeAotSettings.Default); + + private CsProjNativeAotToolchain(NativeAotRuntime runtime, NativeAotSettings settings) + : this(runtime, settings, Resolve(settings, runtime)) { } + + // The build components receive `resolved` (target framework moniker filled in from the runtime); the original + // `settings` is stored for equality and the settings column so an unset moniker is not surfaced as the runtime's. + private CsProjNativeAotToolchain(NativeAotRuntime runtime, NativeAotSettings settings, NativeAotSettings resolved) + : base("CsProjNativeAot", runtime, settings, + new CsProjNativeAotGenerator(resolved), + new DotNetCliPublisher(resolved, GetExtraArguments(settings.RuntimeIdentifier)), + Toolchains.Executor.Instance) + { + } + + // Fills the target framework moniker in from the runtime only when the user left it unset, avoiding both the + // settings copy and the GetTfm() string otherwise. + private static NativeAotSettings Resolve(NativeAotSettings settings, NativeAotRuntime runtime) + => settings.TargetFrameworkMoniker.IsNotBlank() ? settings : settings with { TargetFrameworkMoniker = runtime.GetTfm() }; + + /// Returns a toolchain for the given runtime and settings. + public static CsProjNativeAotToolchain From(NativeAotRuntime runtime, NativeAotSettings settings) + { + if (!settings.Equals(NativeAotSettings.Default)) + return new CsProjNativeAotToolchain(runtime, settings); + + return runtime.Version.Major switch + { + 7 => Net70, + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new CsProjNativeAotToolchain(runtime, settings), + }; + } + + public static string GetExtraArguments(string runtimeIdentifier) => $"-r {runtimeIdentifier}"; +} diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs deleted file mode 100644 index 7ce4b32a5e..0000000000 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs +++ /dev/null @@ -1,363 +0,0 @@ -using BenchmarkDotNet.ConsoleArguments; -using BenchmarkDotNet.Detectors; -using BenchmarkDotNet.Detectors.Cpu; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Loggers; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using System.Text; -using System.Xml; - -namespace BenchmarkDotNet.Toolchains.NativeAot -{ - /// - /// generates new csproj file for self-contained NativeAOT app - /// based on https://github.com/dotnet/corert/blob/7f902d4d8b1c3280e60f5e06c71951a60da173fb/Documentation/how-to-build-and-run-ilcompiler-in-console-shell-prompt.md#compiling-source-to-native-code-using-the-ilcompiler-you-built - /// and https://github.com/dotnet/corert/tree/7f902d4d8b1c3280e60f5e06c71951a60da173fb/samples/HelloWorld#add-corert-to-your-project - /// - public class Generator : CsProjGenerator - { - internal const string NativeAotNuGetFeed = "nativeAotNuGetFeed"; - internal const string GeneratedRdXmlFileName = "bdn_generated.rd.xml"; - - internal Generator(string ilCompilerVersion, - string runtimeFrameworkVersion, - string targetFrameworkMoniker, - string cliPath, - string runtimeIdentifier, - IReadOnlyDictionary feeds, - bool useNuGetClearTag, - bool useTempFolderForRestore, - string packagesRestorePath, - bool rootAllApplicationAssemblies, - bool ilcGenerateStackTraceData, - string ilcOptimizationPreference, - string ilcInstructionSet) - : base(targetFrameworkMoniker, cliPath, GetPackagesDirectoryPath(useTempFolderForRestore, packagesRestorePath), runtimeFrameworkVersion) - { - this.ilCompilerVersion = ilCompilerVersion; - this.runtimeIdentifier = runtimeIdentifier; - this.Feeds = feeds; - this.useNuGetClearTag = useNuGetClearTag; - this.useTempFolderForRestore = useTempFolderForRestore; - this.rootAllApplicationAssemblies = rootAllApplicationAssemblies; - this.ilcGenerateStackTraceData = ilcGenerateStackTraceData; - this.ilcOptimizationPreference = ilcOptimizationPreference; - this.ilcInstructionSet = ilcInstructionSet; - BenchmarkRunCallType = Code.CodeGenBenchmarkRunCallType.Direct; - } - - internal readonly IReadOnlyDictionary Feeds; - private readonly string ilCompilerVersion; - private readonly string runtimeIdentifier; - private readonly bool useNuGetClearTag; - private readonly bool useTempFolderForRestore; - private readonly bool rootAllApplicationAssemblies; - private readonly bool ilcGenerateStackTraceData; - private readonly string ilcOptimizationPreference; - private readonly string ilcInstructionSet; - - protected override string GetExecutableExtension() => OsDetector.ExecutableExtension; - - protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) - => useTempFolderForRestore - ? Path.Combine(Path.GetTempPath(), programName) // store everything in temp to avoid collisions with IDE - : base.GetBuildArtifactsDirectoryPath(buildPartition, programName); - - protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, runtimeIdentifier, "publish"); - - protected override ValueTask GenerateBuildScriptAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) - { - string projectFilePath = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, NullLogger.Instance).FullName; - string extraArguments = NativeAotToolchain.GetExtraArguments(runtimeIdentifier); - - var content = new StringBuilder(300) - .AppendLine($"call {CliPath} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, projectFilePath, extraArguments)}") - .AppendLine($"call {CliPath} {DotNetCliCommand.GetPublishCommand(artifactsPaths, buildPartition, projectFilePath, TargetFrameworkMoniker, extraArguments)}") - .AppendLine($"call {CliPath} {DotNetCliCommand.GetRestoreCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, extraArguments)}") - .AppendLine($"call {CliPath} {DotNetCliCommand.GetPublishCommand(artifactsPaths, buildPartition, artifactsPaths.ProjectFilePath, TargetFrameworkMoniker, extraArguments)}") - .ToString(); - - return new(File.WriteAllTextAsync(artifactsPaths.BuildScriptFilePath, content, cancellationToken)); - } - - // we always want to have a new directory for NuGet packages restore - // to avoid this https://github.com/dotnet/coreclr/blob/master/Documentation/workflow/UsingDotNetCli.md#update-coreclr-using-runtime-nuget-package - // some of the packages are going to contain source code, so they can not be in the subfolder of current solution - // otherwise they would be compiled too (new .csproj include all .cs files from subfolders by default - private static string GetPackagesDirectoryPath(bool useTempFolderForRestore, string packagesRestorePath) - => packagesRestorePath.IsBlank() && useTempFolderForRestore - ? Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()) - : ""; - - protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) - => useTempFolderForRestore && artifactsPaths.PackagesDirectoryName.IsNotBlank() - ? base.GetArtifactsToCleanup(artifactsPaths).Concat([artifactsPaths.PackagesDirectoryName]).ToArray() - : base.GetArtifactsToCleanup(artifactsPaths); - - protected override async ValueTask GenerateNuGetConfigAsync(ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) - { - if (!Feeds.Any()) - return; - - // Skip creating NuGet.config file if clear tag is not specified, and feeds only contain a default nuget.org feed. - if (!useNuGetClearTag && Feeds.Count == 1) - { - if (Feeds.TryGetValue(NativeAotNuGetFeed, out var value) && value == "https://api.nuget.org/v3/index.json") - return; - } - - string content = $""" - - - - {(useNuGetClearTag ? "" : string.Empty)} - {string.Join(Environment.NewLine + " ", Feeds.Select(feed => $""))} - - - """; - - await File.WriteAllTextAsync(artifactsPaths.NuGetConfigPath, content, cancellationToken).ConfigureAwait(false); - } - - protected override async ValueTask GenerateProjectAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) - { - var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).FullName; - - await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, GenerateProjectForNuGetBuild(projectFile, buildPartition, artifactsPaths, logger), cancellationToken).ConfigureAwait(false); - - await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); - await GenerateReflectionFileAsync(artifactsPaths, cancellationToken).ConfigureAwait(false); - } - - private string GenerateProjectForNuGetBuild(string projectFilePath, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) => $""" - - - Exe - {TargetFrameworkMoniker} - {runtimeIdentifier} - {RuntimeFrameworkVersion} - {artifactsPaths.ProgramName} - {artifactsPaths.ProgramName} - true - {buildPartition.Platform.ToConfig()} - False - false - false - false - true - {ilcOptimizationPreference} - {ilcOptimizationPreference} - {GetTrimmingSettings()} - {ilcGenerateStackTraceData} - {ilcGenerateStackTraceData} - false - false - false - {GetInstructionSetSettings(buildPartition)} - - {GetRuntimeSettings(buildPartition.RepresentativeBenchmarkCase.Job.Environment.Gc, buildPartition.Resolver)} - - - - - {GetILCompilerPackageReference()} - - - - {string.Join(Environment.NewLine, GetRdXmlFiles(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger).Select(file => $""))} - - {GetCustomProperties(buildPartition, logger)} - - - latest - - - """; - - private string GetCustomProperties(BuildPartition buildPartition, ILogger logger) - { - var projectFile = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, logger); - var xmlDoc = new XmlDocument(); - xmlDoc.Load(projectFile.FullName); - - (string customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); - return customProperties; - } - - private string GetILCompilerPackageReference() - => ilCompilerVersion.IsBlank() ? "" : $@""; - - private string GetTrimmingSettings() - => rootAllApplicationAssemblies - ? "" - : "full"; - - private string GetInstructionSetSettings(BuildPartition buildPartition) - { - string instructionSet = ilcInstructionSet.IsBlank() - ? GetCurrentInstructionSet(buildPartition.Platform) - : ilcInstructionSet; - - return instructionSet.IsNotBlank() - ? $"{instructionSet}" - : ""; - } - - public IEnumerable GetRdXmlFiles(Type benchmarkTarget, ILogger logger) - { - yield return GeneratedRdXmlFileName; - - var projectFile = GetProjectFilePath(benchmarkTarget, logger); - var projectFileFolder = projectFile.DirectoryName!; - var rdXml = Path.Combine(projectFileFolder, "rd.xml"); - if (File.Exists(rdXml)) - { - yield return rdXml; - } - - foreach (var item in Directory.GetFiles(projectFileFolder, "*.rd.xml")) - { - yield return item; - } - } - - /// - /// mandatory to make it possible to call GC.GetAllocatedBytesForCurrentThread() using reflection (not part of .NET Standard) - /// - private ValueTask GenerateReflectionFileAsync(ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) - { - const string content = """ - - - - - - - - - - - - - - - """; - - string directoryName = Path.GetDirectoryName(artifactsPaths.ProjectFilePath)!; - if (directoryName == null) - throw new InvalidOperationException($"Can't get directory of projectFilePath ('{artifactsPaths.ProjectFilePath}')"); - - return new(File.WriteAllTextAsync(Path.Combine(directoryName, GeneratedRdXmlFileName), content, cancellationToken)); - } - - private string GetCurrentInstructionSet(Platform platform) - => string.Join(",", GetCurrentProcessInstructionSets(platform)); - - // based on https://github.com/dotnet/runtime/tree/v10.0.0-rc.1.25451.107/src/coreclr/tools/Common/JitInterface/ThunkGenerator/InstructionSetDesc.txt - private IEnumerable GetCurrentProcessInstructionSets(Platform platform) - { - if (!ConfigParser.TryParse(TargetFrameworkMoniker, out RuntimeMoniker runtimeMoniker)) - { - throw new NotSupportedException($"Invalid TFM: '{TargetFrameworkMoniker}'"); - } - - // TFM is the MSBuild moniker, not the BDN moniker, so it resolves to Net10_0 instead of NativeAot10_0. - // TODO: Get the correct moniker from the NativeAotRuntime (#2609) - runtimeMoniker += RuntimeMoniker.NativeAot70 - RuntimeMoniker.Net70; - - if (platform == RuntimeInformation.GetCurrentPlatform() // "native" does not support cross-compilation (so does BDN for now) - && runtimeMoniker >= RuntimeMoniker.NativeAot80) - { - yield return "native"; // added in .NET 8 https://github.com/dotnet/runtime/pull/87865 - yield break; - } - - switch (platform) - { - case Platform.X86: - case Platform.X64: - if (HardwareIntrinsics.IsX86BaseSupported) yield return "base"; - if (HardwareIntrinsics.IsX86Sse42Supported) - { - if (runtimeMoniker <= RuntimeMoniker.NativeAot10_0) yield return "sse4.2"; - if (runtimeMoniker <= RuntimeMoniker.NativeAot90) yield return "popcnt"; - } - if (HardwareIntrinsics.IsX86AvxSupported) yield return "avx"; - if (HardwareIntrinsics.IsX86Avx2Supported) - { - yield return "avx2"; - - if (runtimeMoniker <= RuntimeMoniker.NativeAot90) - { - yield return "bmi"; - yield return "bmi2"; - yield return "fma"; - yield return "lzcnt"; - } - } - if (HardwareIntrinsics.IsX86Avx512Supported && (runtimeMoniker > RuntimeMoniker.NativeAot80)) - { - if (runtimeMoniker >= RuntimeMoniker.NativeAot10_0) - { - yield return "avx512"; - } - else - { - yield return "avx512f"; - yield return "avx512f_vl"; - yield return "avx512bw"; - yield return "avx512bw_vl"; - yield return "avx512cd"; - yield return "avx512cd_vl"; - yield return "avx512dq"; - yield return "avx512dq_vl"; - } - } - if (HardwareIntrinsics.IsX86Avx512v2Supported && (runtimeMoniker > RuntimeMoniker.NativeAot80)) - { - if (runtimeMoniker >= RuntimeMoniker.NativeAot10_0) - { - yield return "avx512v2"; - } - else - { - yield return "avx512vbmi"; - yield return "avx512vbmi_vl"; - } - } - if (HardwareIntrinsics.IsX86Avx512v3Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot10_0)) yield return "avx512v3"; - if (HardwareIntrinsics.IsX86Avx10v1Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot90)) yield return "avx10v1"; - if (HardwareIntrinsics.IsX86Avx10v2Supported && (runtimeMoniker >= RuntimeMoniker.NativeAot10_0)) yield return "avx10v2"; - if (HardwareIntrinsics.IsX86AesSupported) - { - yield return "aes"; - if (runtimeMoniker <= RuntimeMoniker.NativeAot90) yield return "pclmul"; - } - if (HardwareIntrinsics.IsX86AvxVnniSupported) yield return "avxvnni"; - if (HardwareIntrinsics.IsX86SerializeSupported && runtimeMoniker > RuntimeMoniker.NativeAot70) yield return "serialize"; // https://github.com/dotnet/BenchmarkDotNet/issues/2463#issuecomment-1809625008 - break; - case Platform.Arm64: - if (HardwareIntrinsics.IsArmBaseSupported) - { - yield return "base"; - yield return "neon"; - } - if (HardwareIntrinsics.IsArmAesSupported) yield return "aes"; - if (HardwareIntrinsics.IsArmCrc32Supported) yield return "crc"; - if (HardwareIntrinsics.IsArmDpSupported) yield return "dotprod"; - if (HardwareIntrinsics.IsArmRdmSupported) yield return "rdma"; - if (HardwareIntrinsics.IsArmSha1Supported) yield return "sha1"; - if (HardwareIntrinsics.IsArmSha256Supported) yield return "sha2"; - break; - default: - yield break; - } - } - } -} diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotSettings.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotSettings.cs new file mode 100644 index 0000000000..ba4972644f --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotSettings.cs @@ -0,0 +1,126 @@ +using System.Collections.Generic; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.NativeAot; + +public sealed record NativeAotSettings : DotNetCliSettings +{ + private const string LocalBuildIlCompilerVersion = "11.0.0-dev"; + + public static readonly NativeAotSettings Default = new(); + + public string RuntimeIdentifier { get; init; } = RuntimeInformation.GetPortableRuntimeIdentifier(); + public bool UseNuGetClearTag { get; init; } + public bool UseTempFolderForRestore { get; init; } + public bool GenerateStackTraceData { get; init; } = true; + public string OptimizationPreference { get; init; } = "Speed"; + public string InstructionSet { get; init; } = ""; + public string IlCompilerVersion { get; private init; } = ""; + /// + /// NuGet feed to restore a specific ILCompiler build from. Mutually exclusive with . + /// For .NET 7+ the ILCompiler is bundled with the SDK, so this is only needed for custom/preview builds. + /// + public string? NuGetFeedUrl { get; private init; } + /// Path to a local build of the ILCompiler packages. Mutually exclusive with . + public DirectoryInfo? LocalIlcPackages { get; private init; } + + public NativeAotSettings() { } + + internal NativeAotSettings(CommandLineOptions options) : base(options) + { + if (options.IlcPackages != null) + { + // Restore the ILCompiler from a local runtime build (the generator also adds the dotnet nightly feed). + LocalIlcPackages = options.IlcPackages; + IlCompilerVersion = LocalBuildIlCompilerVersion; + UseTempFolderForRestore = true; + } + else if (options.ILCompilerVersion.IsNotBlank()) + { + IlCompilerVersion = options.ILCompilerVersion; + } + } + + /// + /// Returns a copy that restores a specific ILCompiler build from a NuGet feed. For .NET 7+ the ILCompiler is bundled + /// with the SDK (use ), so this is only needed for custom/preview builds. + /// + /// the version of Microsoft.DotNet.ILCompiler to use. Empty maps to the latest/bundled version. + /// the NuGet feed to restore from. The default is "https://api.nuget.org/v3/index.json". + public NativeAotSettings WithNuGet(string ilCompilerVersion = "", string nuGetFeedUrl = "https://api.nuget.org/v3/index.json") => this with + { + IlCompilerVersion = ilCompilerVersion, + NuGetFeedUrl = nuGetFeedUrl.IsNotBlank() ? nuGetFeedUrl : null, + LocalIlcPackages = null, + }; + + /// + /// Returns a copy that restores the ILCompiler from a local build of the runtime. + /// See https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/nativeaot.md + /// + /// the path to the shipping packages, example: "C:\runtime\artifacts\packages\Release\Shipping" + public NativeAotSettings WithLocalBuild(DirectoryInfo ilcPackages) + { + if (!ilcPackages.Exists) + throw new DirectoryNotFoundException($"{ilcPackages} provided as {nameof(ilcPackages)} does NOT exist"); + + return this with + { + LocalIlcPackages = ilcPackages, + NuGetFeedUrl = null, + IlCompilerVersion = LocalBuildIlCompilerVersion, + UseTempFolderForRestore = true, + }; + } + + /// + public override void FillSettings(IDictionary settings) + { + base.FillSettings(settings); + settings[nameof(RuntimeIdentifier)] = RuntimeIdentifier; + settings[nameof(UseNuGetClearTag)] = UseNuGetClearTag; + settings[nameof(UseTempFolderForRestore)] = UseTempFolderForRestore; + settings[nameof(GenerateStackTraceData)] = GenerateStackTraceData; + settings[nameof(OptimizationPreference)] = OptimizationPreference; + settings[nameof(InstructionSet)] = InstructionSet; + settings[nameof(IlCompilerVersion)] = IlCompilerVersion; + settings[nameof(NuGetFeedUrl)] = NuGetFeedUrl; + settings[nameof(LocalIlcPackages)] = LocalIlcPackages?.FullName; + } + + // DirectoryInfo compares by reference; compare LocalIlcPackages by value and chain into the base record. + public bool Equals(NativeAotSettings? other) + { + if (other is null || !base.Equals(other)) + return false; + + return RuntimeIdentifier == other.RuntimeIdentifier + && UseNuGetClearTag == other.UseNuGetClearTag + && UseTempFolderForRestore == other.UseTempFolderForRestore + && GenerateStackTraceData == other.GenerateStackTraceData + && OptimizationPreference == other.OptimizationPreference + && InstructionSet == other.InstructionSet + && IlCompilerVersion == other.IlCompilerVersion + && NuGetFeedUrl == other.NuGetFeedUrl + && LocalIlcPackages?.FullName == other.LocalIlcPackages?.FullName; + } + + public override int GetHashCode() + { + var hashCode = new HashCode(); + hashCode.Add(base.GetHashCode()); + hashCode.Add(RuntimeIdentifier); + hashCode.Add(UseNuGetClearTag); + hashCode.Add(UseTempFolderForRestore); + hashCode.Add(GenerateStackTraceData); + hashCode.Add(OptimizationPreference); + hashCode.Add(InstructionSet); + hashCode.Add(IlCompilerVersion); + hashCode.Add(NuGetFeedUrl); + hashCode.Add(LocalIlcPackages?.FullName); + return hashCode.ToHashCode(); + } +} diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs deleted file mode 100644 index d051583253..0000000000 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchain.cs +++ /dev/null @@ -1,82 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Validators; - -namespace BenchmarkDotNet.Toolchains.NativeAot -{ - public class NativeAotToolchain : Toolchain - { - /// - /// compiled as net7.0. - /// - public static readonly IToolchain Net70 = CreateBuilder() - .TargetFrameworkMoniker("net7.0") - .ToToolchain(); - - /// - /// compiled as net8.0. - /// - public static readonly IToolchain Net80 = CreateBuilder() - .TargetFrameworkMoniker("net8.0") - .ToToolchain(); - - /// - /// compiled as net9.0. - /// - public static readonly IToolchain Net90 = CreateBuilder() - .TargetFrameworkMoniker("net9.0") - .ToToolchain(); - - /// - /// compiled as net10.0. - /// - public static readonly IToolchain Net10_0 = CreateBuilder() - .TargetFrameworkMoniker("net10.0") - .ToToolchain(); - - /// - /// compiled as net11.0. - /// - public static readonly IToolchain Net11_0 = CreateBuilder() - .TargetFrameworkMoniker("net11.0") - .ToToolchain(); - - internal NativeAotToolchain(string displayName, - string ilCompilerVersion, - string runtimeFrameworkVersion, string targetFrameworkMoniker, string runtimeIdentifier, - string customDotNetCliPath, string packagesRestorePath, - Dictionary feeds, bool useNuGetClearTag, bool useTempFolderForRestore, - bool rootAllApplicationAssemblies, bool ilcGenerateStackTraceData, - string ilcOptimizationPreference, string ilcInstructionSet) - : base(displayName, - new Generator(ilCompilerVersion, runtimeFrameworkVersion, targetFrameworkMoniker, customDotNetCliPath, - runtimeIdentifier, feeds, useNuGetClearTag, useTempFolderForRestore, packagesRestorePath, - rootAllApplicationAssemblies, ilcGenerateStackTraceData, - ilcOptimizationPreference, ilcInstructionSet), - new DotNetCliPublisher(targetFrameworkMoniker, customDotNetCliPath, GetExtraArguments(runtimeIdentifier)), - new Executor()) - { - CustomDotNetCliPath = customDotNetCliPath; - } - - internal string CustomDotNetCliPath { get; } - - public static NativeAotToolchainBuilder CreateBuilder() => NativeAotToolchainBuilder.Create(); - - public static string GetExtraArguments(string runtimeIdentifier) => $"-r {runtimeIdentifier}"; - - public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) - { - await foreach (var error in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) - { - yield return error; - } - - foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(CustomDotNetCliPath, benchmarkCase)) - { - yield return validationError; - } - } - } -} diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs deleted file mode 100644 index f63e20837c..0000000000 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/NativeAotToolchainBuilder.cs +++ /dev/null @@ -1,147 +0,0 @@ -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Toolchains.DotNetCli; -using JetBrains.Annotations; -using System.Diagnostics.CodeAnalysis; - -namespace BenchmarkDotNet.Toolchains.NativeAot -{ - public class NativeAotToolchainBuilder : CustomDotNetCliToolchainBuilder - { - public static NativeAotToolchainBuilder Create() => new NativeAotToolchainBuilder(); - - private string? ilCompilerVersion; - private string? packagesRestorePath; - // we set those default values on purpose https://github.com/dotnet/BenchmarkDotNet/pull/1057#issuecomment-461832612 - private bool rootAllApplicationAssemblies; - private bool ilcGenerateStackTraceData = true; - private string ilcOptimizationPreference = "Speed"; - private string? ilcInstructionSet; - - /// - /// creates a NativeAOT toolchain targeting NuGet build of Microsoft.DotNet.ILCompiler - /// Based on https://github.com/dotnet/runtimelab/blob/d0a37893a67c125f9b0cd8671846ff7d867df241/samples/HelloWorld/README.md#add-corert-to-your-project - /// - /// the version of Microsoft.DotNet.ILCompiler which should be used. The default is empty which maps to latest version. - /// url to NuGet feed, The default is: "https://api.nuget.org/v3/index.json" - [PublicAPI] - public NativeAotToolchainBuilder UseNuGet(string microsoftDotNetILCompilerVersion = "", string nuGetFeedUrl = "https://api.nuget.org/v3/index.json") - { - ilCompilerVersion = microsoftDotNetILCompilerVersion; - - if (nuGetFeedUrl.IsNotBlank()) - Feeds[Generator.NativeAotNuGetFeed] = nuGetFeedUrl; - - DisplayName(ilCompilerVersion.IsBlank() ? "Latest ILCompiler" : $"ILCompiler {ilCompilerVersion}"); - - return this; - } - - /// - /// creates a NativeAOT toolchain targeting local build of ILCompiler - /// Based on https://github.com/dotnet/runtime/blob/main/docs/workflow/building/coreclr/nativeaot.md - /// - /// the path to shipping packages, example: "C:\runtime\artifacts\packages\Release\Shipping" - [PublicAPI] - public NativeAotToolchainBuilder UseLocalBuild(DirectoryInfo ilcPackages) - { - if (!ilcPackages.Exists) - throw new DirectoryNotFoundException($"{ilcPackages} provided as {nameof(ilcPackages)} does NOT exist"); - - Feeds["local"] = ilcPackages.FullName; - ilCompilerVersion = "11.0.0-dev"; - Feeds["dotnet11"] = "https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet11/nuget/v3/index.json"; - useTempFolderForRestore = true; - DisplayName("local ILCompiler build"); - - return this; - } - - /// - /// The directory to restore packages to (optional). - /// - [PublicAPI] - [SuppressMessage("ReSharper", "ParameterHidesMember")] - public NativeAotToolchainBuilder PackagesRestorePath(string packagesRestorePath) - { - this.packagesRestorePath = packagesRestorePath; - - return this; - } - - /// - /// This controls the compiler behavior where all code in the application assemblies is considered dynamically reachable. - /// This option is disabled by default. - /// Enabling this option (true) has a significant effect on the size of the resulting executable because it prevents removal of unused code that would otherwise happen. - /// - [PublicAPI] - public NativeAotToolchainBuilder RootAllApplicationAssemblies(bool value) - { - rootAllApplicationAssemblies = value; - - return this; - } - - /// - /// This controls generation of stack trace metadata that provides textual names in stack traces. - /// This option is enabled by default. - /// This is for example the text string one gets by calling Exception.ToString() on a caught exception. - /// With this option disabled, stack traces will still be generated, but will be based on reflection metadata alone (they might be less complete). - /// - [PublicAPI] - public NativeAotToolchainBuilder IlcGenerateStackTraceData(bool value) - { - ilcGenerateStackTraceData = value; - - return this; - } - - /// - /// Options related to code generation. - /// - /// "Speed" to favor code execution speed (default), "Size" to favor smaller code size - [PublicAPI] - public NativeAotToolchainBuilder IlcOptimizationPreference(string value = "Speed") - { - ilcOptimizationPreference = value; - - return this; - } - - /// - /// By default, the compiler targets the minimum instruction set supported by the target OS and architecture. - /// This option allows targeting newer instruction sets for better performance. - /// The native binary will require the instruction sets to be supported by the hardware in order to run. - /// For example, `avx2,bmi2,fma,pclmul,popcnt,aes` will produce binary that takes advantage of instruction sets - /// that are typically present on current Intel and AMD processors. - /// - /// Specify empty string ("", not null) to use the defaults. - [PublicAPI] - public NativeAotToolchainBuilder IlcInstructionSet(string value) - { - ilcInstructionSet = value; - - return this; - } - - [PublicAPI] - public override IToolchain ToToolchain() - { - return new NativeAotToolchain( - displayName: displayName!, - ilCompilerVersion: ilCompilerVersion!, - runtimeFrameworkVersion: runtimeFrameworkVersion ?? "", - targetFrameworkMoniker: GetTargetFrameworkMoniker(), - runtimeIdentifier: runtimeIdentifier ?? GetPortableRuntimeIdentifier(), - customDotNetCliPath: customDotNetCliPath ?? "", - packagesRestorePath: packagesRestorePath ?? "", - feeds: Feeds, - useNuGetClearTag: useNuGetClearTag, - useTempFolderForRestore: useTempFolderForRestore, - rootAllApplicationAssemblies: rootAllApplicationAssemblies, - ilcGenerateStackTraceData: ilcGenerateStackTraceData, - ilcOptimizationPreference: ilcOptimizationPreference, - ilcInstructionSet: ilcInstructionSet ?? "" - ); - } - } -} diff --git a/src/BenchmarkDotNet/Toolchains/NetCoreApp/CsProjCoreToolchain.cs b/src/BenchmarkDotNet/Toolchains/NetCoreApp/CsProjCoreToolchain.cs new file mode 100644 index 0000000000..7038ae50e0 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/NetCoreApp/CsProjCoreToolchain.cs @@ -0,0 +1,73 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.NetCoreApp; + +public sealed class CsProjCoreToolchain : CsProjNetToolchain +{ + public static readonly CsProjCoreToolchain NetCoreApp20 = new(CoreRuntime.Core20, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp21 = new(CoreRuntime.Core21, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp22 = new(CoreRuntime.Core22, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp30 = new(CoreRuntime.Core30, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp31 = new(CoreRuntime.Core31, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp50 = new(CoreRuntime.Core50, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp60 = new(CoreRuntime.Core60, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp70 = new(CoreRuntime.Core70, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp80 = new(CoreRuntime.Core80, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp90 = new(CoreRuntime.Core90, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp10_0 = new(CoreRuntime.Core10_0, NetCoreAppSettings.Default); + public static readonly CsProjCoreToolchain NetCoreApp11_0 = new(CoreRuntime.Core11_0, NetCoreAppSettings.Default); + + private CsProjCoreToolchain(CoreRuntime runtime, NetCoreAppSettings settings) + : this(runtime, settings, Resolve(settings, runtime)) { } + + // The build components receive `resolved` (target framework moniker filled in from the runtime); the original + // `settings` is stored for equality and the settings column so an unset moniker is not surfaced as the runtime's. + private CsProjCoreToolchain(Runtime runtime, NetCoreAppSettings settings, NetCoreAppSettings resolved) + : base("CsProjCore", runtime, settings, + new CsProjGenerator(resolved), + new DotNetCliBuilder(resolved), + new DotNetCliExecutor(settings.CliPath)) + { + } + + // Fills the target framework moniker in from the runtime only when the user left it unset, avoiding both the + // settings copy and the GetTfm() string otherwise. Typed to CoreRuntime so its netcoreappX.Y / platform-specific + // GetTfm binds (the MonoCoreRuntime path uses default settings and resolves the moniker inline instead). + private static NetCoreAppSettings Resolve(NetCoreAppSettings settings, CoreRuntime runtime) + => settings.TargetFrameworkMoniker.IsNotBlank() ? settings : settings with { TargetFrameworkMoniker = runtime.GetTfm() }; + + /// Returns a toolchain for the given runtime and settings. + public static CsProjCoreToolchain From(CoreRuntime runtime, NetCoreAppSettings settings) + { + // Platform-specific runtimes (e.g. net8.0-windows) must keep the platform in the generated TFM, + // so they can't reuse the platform-agnostic cached toolchains. + if (!settings.Equals(NetCoreAppSettings.Default) || runtime.IsPlatformSpecific) + return new CsProjCoreToolchain(runtime, settings); + + return (runtime.Version.Major, runtime.Version.Minor) switch + { + (2, 0) => NetCoreApp20, + (2, 1) => NetCoreApp21, + (2, 2) => NetCoreApp22, + (3, 0) => NetCoreApp30, + (3, 1) => NetCoreApp31, + (5, 0) => NetCoreApp50, + (6, 0) => NetCoreApp60, + (7, 0) => NetCoreApp70, + (8, 0) => NetCoreApp80, + (9, 0) => NetCoreApp90, + (10, 0) => NetCoreApp10_0, + (11, 0) => NetCoreApp11_0, + _ => new CsProjCoreToolchain(runtime, settings), + }; + } + + // A .NET SDK with Mono as the default VM uses this plain-dotnet toolchain, but keeps the job's MonoCoreRuntime + // so IToolchain.Runtime matches. See MonoCoreRuntime.GetDefaultToolchain. Always uses default settings, so the + // resolved moniker (plain net{Major}.0 from the base Runtime.GetTfm) is built inline. + internal static CsProjCoreToolchain From(MonoCoreRuntime runtime) + => new(runtime, NetCoreAppSettings.Default, new NetCoreAppSettings { TargetFrameworkMoniker = runtime.GetTfm() }); +} diff --git a/src/BenchmarkDotNet/Toolchains/NetCoreApp/NetCoreAppSettings.cs b/src/BenchmarkDotNet/Toolchains/NetCoreApp/NetCoreAppSettings.cs new file mode 100644 index 0000000000..8b12a76c1d --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/NetCoreApp/NetCoreAppSettings.cs @@ -0,0 +1,13 @@ +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.NetCoreApp; + +public sealed record NetCoreAppSettings : DotNetCliSettings +{ + public static readonly NetCoreAppSettings Default = new(); + + public NetCoreAppSettings() { } + + internal NetCoreAppSettings(CommandLineOptions options) : base(options) { } +} diff --git a/src/BenchmarkDotNet/Toolchains/R2R/CsProjR2RToolchain.cs b/src/BenchmarkDotNet/Toolchains/R2R/CsProjR2RToolchain.cs new file mode 100644 index 0000000000..f2478d0729 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/R2R/CsProjR2RToolchain.cs @@ -0,0 +1,48 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.R2R; + +public sealed class CsProjR2RToolchain : CsProjNetToolchain +{ + public static readonly CsProjR2RToolchain R2R80 = new(R2RRuntime.Net80, R2RSettings.Default); + public static readonly CsProjR2RToolchain R2R90 = new(R2RRuntime.Net90, R2RSettings.Default); + public static readonly CsProjR2RToolchain R2R10_0 = new(R2RRuntime.Net10_0, R2RSettings.Default); + public static readonly CsProjR2RToolchain R2R11_0 = new(R2RRuntime.Net11_0, R2RSettings.Default); + + private CsProjR2RToolchain(R2RRuntime runtime, R2RSettings settings) + : this(runtime, settings, Resolve(settings, runtime)) { } + + // The build components receive `resolved` (target framework moniker filled in from the runtime); the original + // `settings` is stored for equality and the settings column so an unset moniker is not surfaced as the runtime's. + private CsProjR2RToolchain(R2RRuntime runtime, R2RSettings settings, R2RSettings resolved) + : base("CsProjR2R", runtime, settings, + new R2RGenerator(resolved), + new DotNetCliPublisher(resolved), + Toolchains.Executor.Instance) + { + } + + // Fills the target framework moniker in from the runtime only when the user left it unset, avoiding both the + // settings copy and the GetTfm() string otherwise. + private static R2RSettings Resolve(R2RSettings settings, R2RRuntime runtime) + => settings.TargetFrameworkMoniker.IsNotBlank() ? settings : settings with { TargetFrameworkMoniker = runtime.GetTfm() }; + + /// Returns a toolchain for the given runtime and settings. + public static CsProjR2RToolchain From(R2RRuntime runtime, R2RSettings settings) + { + if (!settings.Equals(R2RSettings.Default)) + return new CsProjR2RToolchain(runtime, settings); + + return runtime.Version.Major switch + { + 8 => R2R80, + 9 => R2R90, + 10 => R2R10_0, + 11 => R2R11_0, + _ => new CsProjR2RToolchain(runtime, settings), + }; + } +} diff --git a/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs b/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs index 506d22ac33..683f9d3bb8 100644 --- a/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs @@ -2,24 +2,21 @@ using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; +using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; using System.Text; using System.Xml; namespace BenchmarkDotNet.Toolchains.R2R { - public class R2RGenerator : CsProjGenerator + internal sealed class R2RGenerator : CsProjGenerator { - private readonly string CustomRuntimePack; - private readonly string Crossgen2Pack; + private readonly R2RSettings settings; - public R2RGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string customRuntimePack, string crossgen2Pack) - : base(targetFrameworkMoniker, cliPath, packagesPath) + public R2RGenerator(R2RSettings settings) : base(settings) { - CustomRuntimePack = customRuntimePack; - Crossgen2Pack = crossgen2Pack; + this.settings = settings; BenchmarkRunCallType = Code.CodeGenBenchmarkRunCallType.Direct; } @@ -36,13 +33,13 @@ protected override async ValueTask GenerateProjectAsync(BuildPartition buildPart .Replace("$PLATFORM$", buildPartition.Platform.ToConfig()) .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) .Replace("$CSPROJPATH$", projectFile.FullName) - .Replace("$TFM$", TargetFrameworkMoniker) + .Replace("$TFM$", Settings.TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) .Replace("$SDKNAME$", sdkName) - .Replace("$RUNTIMEPACK$", CustomRuntimePack) - .Replace("$CROSSGEN2PACK$", Crossgen2Pack) - .Replace("$RUNTIMEIDENTIFIER$", CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier()) + .Replace("$RUNTIMEPACK$", settings.CustomRuntimePack?.FullName) + .Replace("$CROSSGEN2PACK$", settings.Crossgen2Pack?.FullName) + .Replace("$RUNTIMEIDENTIFIER$", RuntimeInformation.GetPortableRuntimeIdentifier()) .ToString(); await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); @@ -53,6 +50,6 @@ protected override async ValueTask GenerateProjectAsync(BuildPartition buildPart protected override string GetExecutableExtension() => OsDetector.ExecutableExtension; protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, CustomDotNetCliToolchainBuilder.GetPortableRuntimeIdentifier(), "publish"); + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, RuntimeInformation.GetPortableRuntimeIdentifier(), "publish"); } } diff --git a/src/BenchmarkDotNet/Toolchains/R2R/R2RSettings.cs b/src/BenchmarkDotNet/Toolchains/R2R/R2RSettings.cs new file mode 100644 index 0000000000..7d5cfd605d --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/R2R/R2RSettings.cs @@ -0,0 +1,41 @@ +using System.Collections.Generic; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.R2R; + +public sealed record R2RSettings : DotNetCliSettings +{ + public static readonly R2RSettings Default = new(); + + /// Optional path to a custom runtime pack. + public DirectoryInfo? CustomRuntimePack { get; init; } + /// Optional path to the Crossgen2 pack. + public FileInfo? Crossgen2Pack { get; init; } + + public R2RSettings() { } + + internal R2RSettings(CommandLineOptions options) : base(options) + { + CustomRuntimePack = options.CustomRuntimePack; + Crossgen2Pack = options.AOTCompilerPath; + } + + /// + public override void FillSettings(IDictionary settings) + { + base.FillSettings(settings); + settings[nameof(CustomRuntimePack)] = CustomRuntimePack?.FullName; + settings[nameof(Crossgen2Pack)] = Crossgen2Pack?.FullName; + } + + // FileInfo/DirectoryInfo compare by reference; compare the declared paths by value and chain into the base record. + public bool Equals(R2RSettings? other) + => other is not null + && base.Equals(other) + && CustomRuntimePack?.FullName == other.CustomRuntimePack?.FullName + && Crossgen2Pack?.FullName == other.Crossgen2Pack?.FullName; + + public override int GetHashCode() + => HashCode.Combine(base.GetHashCode(), CustomRuntimePack?.FullName, Crossgen2Pack?.FullName); +} diff --git a/src/BenchmarkDotNet/Toolchains/R2R/R2RToolchain.cs b/src/BenchmarkDotNet/Toolchains/R2R/R2RToolchain.cs deleted file mode 100644 index 45bcb13dc6..0000000000 --- a/src/BenchmarkDotNet/Toolchains/R2R/R2RToolchain.cs +++ /dev/null @@ -1,56 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Validators; -using JetBrains.Annotations; - -namespace BenchmarkDotNet.Toolchains.R2R -{ - [PublicAPI] - public class R2RToolchain : CsProjCoreToolchain, IEquatable - { - [PublicAPI] public static readonly IToolchain R2R80 = From(new NetCoreAppSettings("net8.0", "R2R 8.0")); - [PublicAPI] public static readonly IToolchain R2R90 = From(new NetCoreAppSettings("net9.0", "R2R 9.0")); - [PublicAPI] public static readonly IToolchain R2R10_0 = From(new NetCoreAppSettings("net10.0", "R2R 10.0")); - [PublicAPI] public static readonly IToolchain R2R11_0 = From(new NetCoreAppSettings("net11.0", "R2R 11.0")); - - private readonly string _customDotNetCliPath; - private R2RToolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor, string customDotNetCliPath) - : base(name, generator, builder, executor, customDotNetCliPath) - { - _customDotNetCliPath = customDotNetCliPath; - } - - [PublicAPI] - public static new IToolchain From(NetCoreAppSettings settings) - => new R2RToolchain(settings.Name, - new R2RGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, settings.CustomRuntimePack, settings.AOTCompilerPath), - new DotNetCliPublisher(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath), - new Executor(), - settings.CustomDotNetCliPath); - - public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) - { - foreach (var validationError in DotNetSdkValidator.ValidateCoreSdks(_customDotNetCliPath, benchmarkCase)) - { - yield return validationError; - } - } - - public override bool Equals(object? obj) => obj is R2RToolchain typed && Equals(typed); - - public bool Equals(R2RToolchain? other) - { - if (ReferenceEquals(this, other)) - return true; - - if (other is null) - return false; - - return Generator.Equals(other.Generator); - } - - public override int GetHashCode() => Generator.GetHashCode(); - } -} diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynBuilder.cs similarity index 98% rename from src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs rename to src/BenchmarkDotNet/Toolchains/Roslyn/RoslynBuilder.cs index 496e2add38..f78ce0257c 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Builder.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynBuilder.cs @@ -15,11 +15,11 @@ namespace BenchmarkDotNet.Toolchains.Roslyn { [PublicAPI] - public class Builder : IBuilder + public class RoslynBuilder : IBuilder { private const string MissingReferenceError = "CS0012"; - public static readonly IBuilder Instance = new Builder(); + public static readonly IBuilder Instance = new RoslynBuilder(); private static readonly Lazy FrameworkAssembliesMetadata = new Lazy(GetFrameworkAssembliesMetadata); @@ -57,7 +57,7 @@ private async ValueTask Build(GenerateResult generateResult, BuildP compilationOptions = compilationOptions.WithIgnoreCorLibraryDuplicatedTypes(); - var references = Generator + var references = RoslynGenerator .GetAllReferences(buildPartition.Benchmarks[0]) .Select(assembly => AssemblyMetadata.CreateFromFile(assembly.Location)) .Concat(FrameworkAssembliesMetadata.Value) diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynGenerator.cs similarity index 95% rename from src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs rename to src/BenchmarkDotNet/Toolchains/Roslyn/RoslynGenerator.cs index 30a5247fac..37c92e4226 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynGenerator.cs @@ -1,19 +1,17 @@ using BenchmarkDotNet.Detectors; -using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Running; -using JetBrains.Annotations; using System.Reflection; namespace BenchmarkDotNet.Toolchains.Roslyn { - [PublicAPI] - public class Generator : GeneratorBase + public class RoslynGenerator : GeneratorBase { + public static readonly RoslynGenerator Instance = new(); + protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPartition, string programName) => Path.GetDirectoryName(buildPartition.AssemblyLocation)!; - [PublicAPI] protected override string[] GetArtifactsToCleanup(ArtifactsPaths artifactsPaths) => [ artifactsPaths.ProgramCodePath, diff --git a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs index 5e63a9ef20..91ebae3814 100644 --- a/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Roslyn/RoslynToolchain.cs @@ -1,26 +1,15 @@ using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; -using JetBrains.Annotations; namespace BenchmarkDotNet.Toolchains.Roslyn { - /// - /// Build a benchmark program with the Roslyn compiler. - /// - [PublicAPI] - public class RoslynToolchain : Toolchain + public abstract class RoslynToolchain(string name, Runtime runtime, IBuilder builder, IExecutor executor) + : Toolchain(name, runtime, RoslynGenerator.Instance, builder, executor) { - public static readonly IToolchain Instance = new RoslynToolchain(); - - [PublicAPI] - public RoslynToolchain() : base("Roslyn", new Generator(), Roslyn.Builder.Instance, new Executor()) - { - } - - [PublicAPI] public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) { await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) @@ -31,14 +20,7 @@ public override async IAsyncEnumerable ValidateAsync(BenchmarkC if (!(RuntimeInformation.IsFullFramework || RuntimeInformation.IsOldMono)) { yield return new ValidationError(true, - "The Roslyn toolchain is only supported on .NET Framework and legacy Mono", - benchmarkCase); - } - - if (benchmarkCase.Job.ResolveValue(GcMode.RetainVmCharacteristic, resolver)) - { - yield return new ValidationError(true, - $"Currently App.config does not support RetainVM option, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", + $"{GetType().Name} is only supported on .NET Framework and legacy Mono", benchmarkCase); } @@ -46,9 +28,16 @@ public override async IAsyncEnumerable ValidateAsync(BenchmarkC && benchmarkCase.Job.ResolveValue(InfrastructureMode.BuildConfigurationCharacteristic, resolver) != InfrastructureMode.ReleaseConfigurationName) { yield return new ValidationError(true, - "The Roslyn toolchain does not allow to rebuild source project, so defining custom build configuration makes no sense", + $"{GetType().Name} does not allow to rebuild source project, so defining custom build configuration makes no sense", benchmarkCase); } } + + public override bool Equals(object? obj) + => obj is RoslynToolchain other + && other.GetType() == GetType() + && Runtime.Equals(other.Runtime); + + public override int GetHashCode() => HashCode.Combine(GetType(), Runtime); } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/RuntimeTfmExtensions.cs b/src/BenchmarkDotNet/Toolchains/RuntimeTfmExtensions.cs new file mode 100644 index 0000000000..bd1b0aca51 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/RuntimeTfmExtensions.cs @@ -0,0 +1,29 @@ +using BenchmarkDotNet.Environments; + +namespace BenchmarkDotNet.Toolchains; + +/// +/// Derives the target framework moniker (e.g. net8.0, net472, netcoreapp3.1) that SDK-based +/// toolchains build against. This is a build detail owned by the toolchain layer, not a property of the runtime. +/// Callers bind to the overload matching their statically-known runtime type — no runtime type dispatch — so a +/// toolchain that already holds a concrete runtime gets the right moniker directly; the +/// overload is the fallback for the runtimes whose default moniker is just net{Major}.{Minor}. +/// +internal static class RuntimeTfmExtensions +{ + internal static string GetTfm(this ClrRuntime runtime) + => runtime.Version.Build > 0 + ? $"net{runtime.Version.Major}{runtime.Version.Minor}{runtime.Version.Build}" + : $"net{runtime.Version.Major}{runtime.Version.Minor}"; + + internal static string GetTfm(this CoreRuntime runtime) + // .NET Core < 5 uses the netcoreappX.Y moniker; 5+ never has a non-zero minor version. + => runtime.Version.Major < 5 ? $"netcoreapp{runtime.Version.Major}.{runtime.Version.Minor}" + // The platform is passed through as given: it carries a version that selects a different platform surface + // (net10.0-windows10.0.19041.0), and the SDK canonicalizes the case itself. + : runtime.IsPlatformSpecific ? $"net{runtime.Version.Major}.0-{runtime.Platform}" + : $"net{runtime.Version.Major}.0"; + + internal static string GetTfm(this Runtime runtime) + => $"net{runtime.Version!.Major}.{runtime.Version.Minor}"; +} diff --git a/src/BenchmarkDotNet/Toolchains/Toolchain.cs b/src/BenchmarkDotNet/Toolchains/Toolchain.cs index 2dcb3ad4f6..6db8d2ef51 100644 --- a/src/BenchmarkDotNet/Toolchains/Toolchain.cs +++ b/src/BenchmarkDotNet/Toolchains/Toolchain.cs @@ -1,61 +1,34 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.Toolchains { - public class Toolchain : IToolchain + public abstract class Toolchain(string name, Runtime runtime, IGenerator generator, IBuilder builder, IExecutor executor) : IToolchain { - public string Name { get; } + public Runtime Runtime { get; } = runtime; - public IGenerator Generator { get; } + public IGenerator Generator { get; } = generator; - public IBuilder Builder { get; } + public IBuilder Builder { get; } = builder; - public IExecutor Executor { get; } + public IExecutor Executor { get; } = executor; public virtual bool IsInProcess => false; - public Toolchain(string name, IGenerator generator, IBuilder builder, IExecutor executor) - { - Name = name; - Generator = generator; - Builder = builder; - Executor = executor; - } - public virtual async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) { - var runtime = benchmarkCase.Job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, resolver); var jit = benchmarkCase.Job.ResolveValue(EnvironmentMode.JitCharacteristic, resolver); - if (!(runtime is MonoRuntime) && jit == Jit.Llvm) + if (jit == Jit.Llvm && Runtime is not (LegacyMonoRuntime or MonoCoreRuntime)) { yield return new ValidationError(true, $"Llvm is supported only for Mono, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", benchmarkCase); } - - if (runtime is MonoRuntime mono && !mono.IsDotNetBuiltIn && !benchmarkCase.GetToolchain().IsInProcess) - { - if (mono.CustomPath.IsBlank() && !HostEnvironmentInfo.GetCurrent().IsMonoInstalled.Value) - { - yield return new ValidationError(true, - $"Mono is not installed or added to PATH, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - } - - if (mono.CustomPath.IsNotBlank() && !File.Exists(mono.CustomPath)) - { - yield return new ValidationError(true, - $"We could not find Mono in provided path ({mono.CustomPath}), benchmark '{benchmarkCase.DisplayInfo}' will not be executed", - benchmarkCase); - } - } } - public override string ToString() => Name; + public override string ToString() => Runtime.Version == null ? name : $"{name} {Runtime.Version}"; } -} \ No newline at end of file +} diff --git a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs b/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs deleted file mode 100644 index 4dd2d10c09..0000000000 --- a/src/BenchmarkDotNet/Toolchains/ToolchainExtensions.cs +++ /dev/null @@ -1,227 +0,0 @@ -using BenchmarkDotNet.Detectors; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Portability; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.InProcess.Emit; -using BenchmarkDotNet.Toolchains.InProcess.NoEmit; -using BenchmarkDotNet.Toolchains.Mono; -using BenchmarkDotNet.Toolchains.MonoWasm; -using BenchmarkDotNet.Toolchains.NativeAot; -using BenchmarkDotNet.Toolchains.R2R; -using BenchmarkDotNet.Toolchains.Roslyn; - -namespace BenchmarkDotNet.Toolchains -{ - internal static class ToolchainExtensions - { - internal static IToolchain GetToolchain(this BenchmarkCase benchmarkCase) - => benchmarkCase.Job.Infrastructure.TryGetToolchain(out var toolchain) - ? toolchain - : GetToolchain( - benchmarkCase.GetRuntime(), - benchmarkCase.Descriptor, - benchmarkCase.Job.HasDynamicBuildCharacteristic(), - benchmarkCase.Job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic) - ); - - internal static IToolchain GetToolchain(this Job job) - => job.Infrastructure.TryGetToolchain(out var toolchain) - ? toolchain - : GetToolchain( - job.ResolveValue(EnvironmentMode.RuntimeCharacteristic, EnvironmentResolver.Instance)!, - null, - job.HasDynamicBuildCharacteristic(), - job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic) - ); - - internal static IToolchain GetToolchain(this Runtime runtime, Descriptor? descriptor = null, bool preferMsBuildToolchains = false, bool isRuntimeExplicit = false) - { - switch (runtime) - { - case ClrRuntime clrRuntime: - bool UseRoslyn() - => !isRuntimeExplicit - || runtime.MsBuildMoniker == ClrRuntime.GetTargetOrCurrentVersion(descriptor?.Type.Assembly).MsBuildMoniker; - - if (!preferMsBuildToolchains && RuntimeInformation.IsFullFramework && UseRoslyn()) - return RoslynToolchain.Instance; - - return clrRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized - ? GetToolchain(clrRuntime.RuntimeMoniker) - : CsProjClassicNetToolchain.From(clrRuntime.MsBuildMoniker); - - case MonoRuntime mono: - if (OsDetector.IsAndroid()) - return InProcessEmitToolchain.Default; - if (OsDetector.IsIOS()) - return InProcessNoEmitToolchain.Default; - if (mono.AotArgs.IsNotBlank()) - return MonoAotToolchain.Instance; - if (mono.IsDotNetBuiltIn) - if (RuntimeInformation.IsNewMono) - { - // It's a .NET SDK with Mono as default VM. - // Publishing self-contained apps might not work like in https://github.com/dotnet/performance/issues/2787. - // In such case, we are going to use default .NET toolchain that is just going to perform dotnet build, - // which internally will result in creating Mono-based app. - return mono.RuntimeMoniker switch - { - RuntimeMoniker.Mono60 => GetToolchain(RuntimeMoniker.Net60), - RuntimeMoniker.Mono70 => GetToolchain(RuntimeMoniker.Net70), - RuntimeMoniker.Mono80 => GetToolchain(RuntimeMoniker.Net80), - RuntimeMoniker.Mono90 => GetToolchain(RuntimeMoniker.Net90), - RuntimeMoniker.Mono10_0 => GetToolchain(RuntimeMoniker.Net10_0), - RuntimeMoniker.Mono11_0 => GetToolchain(RuntimeMoniker.Net11_0), - _ => CsProjCoreToolchain.From(new NetCoreAppSettings(mono.MsBuildMoniker, mono.Name)) - }; - } - else - { - return MonoToolchain.From( - new NetCoreAppSettings(targetFrameworkMoniker: mono.MsBuildMoniker, name: mono.Name)); - } - - return RoslynToolchain.Instance; - - case CoreRuntime coreRuntime: - if (descriptor != null && descriptor.Type.Assembly.IsLinqPad()) - return InProcessEmitToolchain.Default; - if (coreRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized && !coreRuntime.IsPlatformSpecific) - return GetToolchain(coreRuntime.RuntimeMoniker); - - return CsProjCoreToolchain.From(new NetCoreAppSettings(coreRuntime.MsBuildMoniker, coreRuntime.Name)); - - case NativeAotRuntime nativeAotRuntime: - return nativeAotRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized - ? GetToolchain(nativeAotRuntime.RuntimeMoniker) - : NativeAotToolchain.CreateBuilder().UseNuGet().TargetFrameworkMoniker(nativeAotRuntime.MsBuildMoniker).ToToolchain(); - - case WasmRuntime wasmRuntime: - return WasmToolchain.From(new NetCoreAppSettings(targetFrameworkMoniker: wasmRuntime.MsBuildMoniker, name: wasmRuntime.Name)); - - case R2RRuntime r2rRuntime: - if (r2rRuntime.RuntimeMoniker != RuntimeMoniker.NotRecognized) - return GetToolchain(r2rRuntime.RuntimeMoniker); - - return CsProjCoreToolchain.From(new NetCoreAppSettings(r2rRuntime.MsBuildMoniker, r2rRuntime.Name)); - - default: - throw new ArgumentOutOfRangeException(nameof(runtime), runtime, "Runtime not supported"); - } - } - - private static IToolchain GetToolchain(RuntimeMoniker runtimeMoniker) - { - switch (runtimeMoniker) - { - case RuntimeMoniker.Net461: - return CsProjClassicNetToolchain.Net461; - - case RuntimeMoniker.Net462: - return CsProjClassicNetToolchain.Net462; - - case RuntimeMoniker.Net47: - return CsProjClassicNetToolchain.Net47; - - case RuntimeMoniker.Net471: - return CsProjClassicNetToolchain.Net471; - - case RuntimeMoniker.Net472: - return CsProjClassicNetToolchain.Net472; - - case RuntimeMoniker.Net48: - return CsProjClassicNetToolchain.Net48; - - case RuntimeMoniker.Net481: - return CsProjClassicNetToolchain.Net481; - - case RuntimeMoniker.NetCoreApp20: - return CsProjCoreToolchain.NetCoreApp20; - - case RuntimeMoniker.NetCoreApp21: - return CsProjCoreToolchain.NetCoreApp21; - - case RuntimeMoniker.NetCoreApp22: - return CsProjCoreToolchain.NetCoreApp22; - - case RuntimeMoniker.NetCoreApp30: - return CsProjCoreToolchain.NetCoreApp30; - - case RuntimeMoniker.NetCoreApp31: - return CsProjCoreToolchain.NetCoreApp31; - case RuntimeMoniker.Net50: - return CsProjCoreToolchain.NetCoreApp50; - - case RuntimeMoniker.Net60: - return CsProjCoreToolchain.NetCoreApp60; - - case RuntimeMoniker.Net70: - return CsProjCoreToolchain.NetCoreApp70; - - case RuntimeMoniker.Net80: - return CsProjCoreToolchain.NetCoreApp80; - - case RuntimeMoniker.Net90: - return CsProjCoreToolchain.NetCoreApp90; - - case RuntimeMoniker.Net10_0: - return CsProjCoreToolchain.NetCoreApp10_0; - - case RuntimeMoniker.Net11_0: - return CsProjCoreToolchain.NetCoreApp11_0; - - case RuntimeMoniker.NativeAot70: - return NativeAotToolchain.Net70; - - case RuntimeMoniker.NativeAot80: - return NativeAotToolchain.Net80; - - case RuntimeMoniker.NativeAot90: - return NativeAotToolchain.Net90; - - case RuntimeMoniker.NativeAot10_0: - return NativeAotToolchain.Net10_0; - - case RuntimeMoniker.NativeAot11_0: - return NativeAotToolchain.Net11_0; - - case RuntimeMoniker.Mono60: - return MonoToolchain.Mono60; - - case RuntimeMoniker.Mono70: - return MonoToolchain.Mono70; - - case RuntimeMoniker.Mono80: - return MonoToolchain.Mono80; - - case RuntimeMoniker.Mono90: - return MonoToolchain.Mono90; - - case RuntimeMoniker.Mono10_0: - return MonoToolchain.Mono10_0; - - case RuntimeMoniker.Mono11_0: - return MonoToolchain.Mono11_0; - - case RuntimeMoniker.R2R80: - return R2RToolchain.R2R80; - - case RuntimeMoniker.R2R90: - return R2RToolchain.R2R90; - - case RuntimeMoniker.R2R10_0: - return R2RToolchain.R2R10_0; - - case RuntimeMoniker.R2R11_0: - return R2RToolchain.R2R11_0; - - default: - throw new ArgumentOutOfRangeException(nameof(runtimeMoniker), runtimeMoniker, "RuntimeMoniker not supported"); - } - } - } -} diff --git a/src/BenchmarkDotNet/Toolchains/Wasm/CsProjCoreWasmToolchain.cs b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjCoreWasmToolchain.cs new file mode 100644 index 0000000000..1a0c9866c8 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjCoreWasmToolchain.cs @@ -0,0 +1,16 @@ +using BenchmarkDotNet.Environments; +using System.ComponentModel; + +namespace BenchmarkDotNet.Toolchains.Wasm; + +/// Toolchain that builds and runs benchmarks on CoreCLR WebAssembly. +[EditorBrowsable(EditorBrowsableState.Never)] // WebAssembly with CoreCLR is still experimental. This is only used by dotnet contributors and dotnet/performance until official support is released. +public sealed class CsProjCoreWasmToolchain : CsProjWasmToolchain +{ + private CsProjCoreWasmToolchain(CoreWasmRuntime runtime, WasmSettings settings) + : base("CoreWasm", runtime, settings, aot: false, useCoreClrRuntime: true) { } + + /// Creates a toolchain for the given runtime and settings. + public static CsProjCoreWasmToolchain From(CoreWasmRuntime runtime, WasmSettings settings) + => new(runtime, settings); +} diff --git a/src/BenchmarkDotNet/Toolchains/Wasm/CsProjMonoWasmAotToolchain.cs b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjMonoWasmAotToolchain.cs new file mode 100644 index 0000000000..83ad6e8558 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjMonoWasmAotToolchain.cs @@ -0,0 +1,31 @@ +using BenchmarkDotNet.Environments; + +namespace BenchmarkDotNet.Toolchains.Wasm; + +/// Toolchain that builds and runs benchmarks on Mono WebAssembly AOT. +public sealed class CsProjMonoWasmAotToolchain : CsProjWasmToolchain +{ + public static readonly CsProjMonoWasmAotToolchain Net80 = new(MonoWasmAotRuntime.Net80, WasmSettings.Default); + public static readonly CsProjMonoWasmAotToolchain Net90 = new(MonoWasmAotRuntime.Net90, WasmSettings.Default); + public static readonly CsProjMonoWasmAotToolchain Net10_0 = new(MonoWasmAotRuntime.Net10_0, WasmSettings.Default); + public static readonly CsProjMonoWasmAotToolchain Net11_0 = new(MonoWasmAotRuntime.Net11_0, WasmSettings.Default); + + private CsProjMonoWasmAotToolchain(MonoWasmAotRuntime runtime, WasmSettings settings) + : base("MonoWasmAot", runtime, settings, aot: true, useCoreClrRuntime: false) { } + + /// Returns a toolchain for the given runtime and settings. + public static CsProjMonoWasmAotToolchain From(MonoWasmAotRuntime runtime, WasmSettings settings) + { + if (!settings.Equals(WasmSettings.Default)) + return new CsProjMonoWasmAotToolchain(runtime, settings); + + return runtime.Version.Major switch + { + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new CsProjMonoWasmAotToolchain(runtime, settings), + }; + } +} diff --git a/src/BenchmarkDotNet/Toolchains/Wasm/CsProjMonoWasmToolchain.cs b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjMonoWasmToolchain.cs new file mode 100644 index 0000000000..a33f2b521e --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjMonoWasmToolchain.cs @@ -0,0 +1,31 @@ +using BenchmarkDotNet.Environments; + +namespace BenchmarkDotNet.Toolchains.Wasm; + +/// Toolchain that builds and runs benchmarks on the Mono WebAssembly interpreter. +public sealed class CsProjMonoWasmToolchain : CsProjWasmToolchain +{ + public static readonly CsProjMonoWasmToolchain Net80 = new(MonoWasmRuntime.Net80, WasmSettings.Default); + public static readonly CsProjMonoWasmToolchain Net90 = new(MonoWasmRuntime.Net90, WasmSettings.Default); + public static readonly CsProjMonoWasmToolchain Net10_0 = new(MonoWasmRuntime.Net10_0, WasmSettings.Default); + public static readonly CsProjMonoWasmToolchain Net11_0 = new(MonoWasmRuntime.Net11_0, WasmSettings.Default); + + private CsProjMonoWasmToolchain(MonoWasmRuntime runtime, WasmSettings settings) + : base("MonoWasm", runtime, settings, aot: false, useCoreClrRuntime: false) { } + + /// Returns a toolchain for the given runtime and settings. + public static CsProjMonoWasmToolchain From(MonoWasmRuntime runtime, WasmSettings settings) + { + if (!settings.Equals(WasmSettings.Default)) + return new CsProjMonoWasmToolchain(runtime, settings); + + return runtime.Version.Major switch + { + 8 => Net80, + 9 => Net90, + 10 => Net10_0, + 11 => Net11_0, + _ => new CsProjMonoWasmToolchain(runtime, settings), + }; + } +} diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjWasmGenerator.cs similarity index 82% rename from src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs rename to src/BenchmarkDotNet/Toolchains/Wasm/CsProjWasmGenerator.cs index 2692eb6f1b..1755b16e9b 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjWasmGenerator.cs @@ -1,4 +1,3 @@ -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Loggers; @@ -7,16 +6,19 @@ using System.Text; using System.Xml; -namespace BenchmarkDotNet.Toolchains.MonoWasm +namespace BenchmarkDotNet.Toolchains.Wasm { - public class WasmGenerator : CsProjGenerator + public class CsProjWasmGenerator : CsProjGenerator { - private readonly string CustomRuntimePack; + private readonly WasmSettings settings; + private readonly bool aot; + private readonly bool useCoreClrRuntime; - public WasmGenerator(string targetFrameworkMoniker, string cliPath, string packagesPath, string customRuntimePack, bool aot) - : base(targetFrameworkMoniker, cliPath, packagesPath) + public CsProjWasmGenerator(WasmSettings settings, bool aot, bool useCoreClrRuntime) : base(settings) { - CustomRuntimePack = customRuntimePack; + this.settings = settings; + this.aot = aot; + this.useCoreClrRuntime = useCoreClrRuntime; EntryPointType = Code.CodeGenEntryPointType.Asynchronous; BenchmarkRunCallType = aot ? Code.CodeGenBenchmarkRunCallType.Direct : Code.CodeGenBenchmarkRunCallType.Reflection; } @@ -25,7 +27,7 @@ protected override async ValueTask GenerateProjectAsync(BuildPartition buildPart { var targetMainJsPath = GetExecutablePath(Path.GetDirectoryName(artifactsPaths.ProjectFilePath)!, ""); - if (buildPartition.Runtime.IsAOT) + if (aot) { await GenerateProjectFileAsync(buildPartition, artifactsPaths, aot: true, logger, targetMainJsPath, cancellationToken).ConfigureAwait(false); @@ -40,7 +42,7 @@ await ResourceHelper.LoadTemplateAsync(linkDescriptionFileName, cancellationToke await GenerateProjectFileAsync(buildPartition, artifactsPaths, aot: false, logger: logger, targetMainJsPath, cancellationToken).ConfigureAwait(false); } - await GenerateMainJS(((WasmRuntime)buildPartition.Runtime).MainJsTemplate, targetMainJsPath, cancellationToken).ConfigureAwait(false); + await GenerateMainJS(settings.MainJsTemplate, targetMainJsPath, cancellationToken).ConfigureAwait(false); } protected async ValueTask GenerateProjectFileAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, bool aot, ILogger logger, string targetMainJsPath, CancellationToken cancellationToken) @@ -48,8 +50,6 @@ protected async ValueTask GenerateProjectFileAsync(BuildPartition buildPartition BenchmarkCase benchmark = buildPartition.RepresentativeBenchmarkCase; var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); - WasmRuntime runtime = (WasmRuntime)buildPartition.Runtime; - var xmlDoc = new XmlDocument(); xmlDoc.Load(projectFile.FullName); var (customProperties, _) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); @@ -59,7 +59,7 @@ protected async ValueTask GenerateProjectFileAsync(BuildPartition buildPartition // - UseMonoRuntime=false: resolves CoreCLR runtime pack instead of Mono // - WasmBuildNative=false: avoids requiring wasm-tools workload // - WasmEnableWebcil=false: CoreCLR doesn't support webcil format - string coreclrOverrides = runtime.RuntimeFlavor == RuntimeFlavor.Mono + string coreclrOverrides = !useCoreClrRuntime ? string.Empty : """ @@ -76,14 +76,13 @@ protected async ValueTask GenerateProjectFileAsync(BuildPartition buildPartition .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) .Replace("$RUN_AOT$", aot.ToString().ToLower()) .Replace("$CSPROJPATH$", projectFile.FullName) - .Replace("$TFM$", TargetFrameworkMoniker) + .Replace("$TFM$", Settings.TargetFrameworkMoniker) .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) .Replace("$COPIEDSETTINGS$", customProperties) .Replace("$SDKNAME$", sdkName) - .Replace("$TARGET$", CustomRuntimePack.IsNotBlank() ? "PublishWithCustomRuntimePack" : "Publish") .Replace("$MAINJS$", targetMainJsPath) .Replace("$CORECLR_OVERRIDES$", coreclrOverrides) - .ToString(); + .ToString(); await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); @@ -103,6 +102,6 @@ protected async ValueTask GenerateMainJS(FileInfo? mainJsTemplate, string target protected override string GetExecutablePath(string binariesDirectoryPath, string programName) => Path.Combine(binariesDirectoryPath, "wwwroot", "main.mjs"); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) - => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker, "publish"); + => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, Settings.TargetFrameworkMoniker, "publish"); } } diff --git a/src/BenchmarkDotNet/Toolchains/Wasm/CsProjWasmToolchain.cs b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjWasmToolchain.cs new file mode 100644 index 0000000000..40dc8aa6b4 --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Wasm/CsProjWasmToolchain.cs @@ -0,0 +1,49 @@ +using System.Collections.Generic; +using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.DotNetCli; +using BenchmarkDotNet.Validators; + +namespace BenchmarkDotNet.Toolchains.Wasm; + +public abstract class CsProjWasmToolchain : CsProjNetToolchain +{ + private readonly WasmSettings settings; + + protected CsProjWasmToolchain(string name, WasmRuntime runtime, WasmSettings settings, bool aot, bool useCoreClrRuntime) + : this(name, runtime, settings, Resolve(settings, runtime), aot, useCoreClrRuntime) { } + + // The build components receive `resolved` (target framework moniker filled in from the runtime); the original + // `settings` is stored for equality and the settings column so an unset moniker is not surfaced as the runtime's. + // The executor does not use the moniker, so it keeps the original settings. + private CsProjWasmToolchain(string name, WasmRuntime runtime, WasmSettings settings, WasmSettings resolved, bool aot, bool useCoreClrRuntime) + : base(name, runtime, settings, + new CsProjWasmGenerator(resolved, aot, useCoreClrRuntime), + new DotNetCliPublisher(resolved, logOutput: aot), + new WasmExecutor(settings)) + => this.settings = settings; + + // Fills the target framework moniker in from the runtime only when the user left it unset, avoiding both the + // settings copy and the GetTfm() string otherwise. + private static WasmSettings Resolve(WasmSettings settings, WasmRuntime runtime) + => settings.TargetFrameworkMoniker.IsNotBlank() ? settings : settings with { TargetFrameworkMoniker = runtime.GetTfm() }; + + public override async IAsyncEnumerable ValidateAsync(BenchmarkCase benchmarkCase, IResolver resolver) + { + await foreach (var validationError in base.ValidateAsync(benchmarkCase, resolver).ConfigureAwait(false)) + { + yield return validationError; + } + + if (!ProcessHelper.TryResolveExecutableInPath(settings.JavaScriptEngine, out _)) + { + yield return new ValidationError(true, + $"The JavaScript engine '{settings.JavaScriptEngine}' was not found. Make sure it is installed and on your PATH.", + benchmarkCase); + } + } +} diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs b/src/BenchmarkDotNet/Toolchains/Wasm/WasmExecutor.cs similarity index 82% rename from src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs rename to src/BenchmarkDotNet/Toolchains/Wasm/WasmExecutor.cs index ed7fd12c6d..d28cbc8656 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmExecutor.cs +++ b/src/BenchmarkDotNet/Toolchains/Wasm/WasmExecutor.cs @@ -1,7 +1,6 @@ using BenchmarkDotNet.Characteristics; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; @@ -11,9 +10,9 @@ using BenchmarkDotNet.Toolchains.Results; using System.Diagnostics; -namespace BenchmarkDotNet.Toolchains.MonoWasm +namespace BenchmarkDotNet.Toolchains.Wasm { - internal class WasmExecutor : IExecutor + public class WasmExecutor(WasmSettings settings) : IExecutor { private sealed class ProcessListener(IpcListener listener, Process process) : IDisposable { @@ -41,7 +40,7 @@ public async ValueTask ExecuteAsync(ExecuteParameters executePara executeParameters.DiagnoserRunMode, cancellationToken).ConfigureAwait(false); } - private static async ValueTask ProbeWebSocketSupportAsync(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, IResolver resolver, CancellationToken cancellationToken) + private async ValueTask ProbeWebSocketSupportAsync(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, IResolver resolver, CancellationToken cancellationToken) { // Check if the JavaScript runtime supports WebSocket using var probeProcess = CreateProcess(benchmarkCase, artifactsPaths, "--getSupportsWebSocket", resolver); @@ -58,7 +57,7 @@ private static async ValueTask ProbeWebSocketSupportAsync(BenchmarkCase be } finally { - if (!probeProcess.WaitForExit(milliseconds: (int)ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) + if (!probeProcess.WaitForExit(milliseconds: (int) ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) { probeProcess.KillTree(); } @@ -78,16 +77,14 @@ private static async ValueTask ProbeWebSocketSupportAsync(BenchmarkCase be return false; } - private static async ValueTask CreateProcessListenerAsync( + private async ValueTask CreateProcessListenerAsync( BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ArtifactsPaths artifactsPaths, IResolver resolver, Diagnosers.RunMode diagnoserRunMode, CancellationToken cancellationToken) { - WasmRuntime runtime = (WasmRuntime)benchmarkCase.GetRuntime(); - - bool useWebSocket = runtime.IpcType == WasmIpcType.Auto + bool useWebSocket = settings.IpcType == WasmIpcType.Auto // Probe the JavaScript runtime to check if it supports WebSocket ? await ProbeWebSocketSupportAsync(benchmarkCase, artifactsPaths, resolver, cancellationToken).ConfigureAwait(false) - : runtime.IpcType == WasmIpcType.WebSocket; + : settings.IpcType == WasmIpcType.WebSocket; IpcListener listener; string args; @@ -131,7 +128,7 @@ private static async ValueTask CreateProcessListenerAsync( return new ProcessListener(listener, process); } - private static async ValueTask Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, Loggers.ILogger logger, ArtifactsPaths artifactsPaths, + private async ValueTask Execute(BenchmarkCase benchmarkCase, BenchmarkId benchmarkId, ILogger logger, ArtifactsPaths artifactsPaths, IDiagnoser diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, IResolver resolver, int launchIndex, Diagnosers.RunMode diagnoserRunMode, CancellationToken cancellationToken) { @@ -147,7 +144,7 @@ private static async ValueTask Execute(BenchmarkCase benchmarkCas if (isFileBasedIpc) { - ((FileStdOutListener)processListener.Listener).AttachProcessOutputReader(processOutputReader); + ((FileStdOutListener) processListener.Listener).AttachProcessOutputReader(processOutputReader); } await diagnoser.HandleAsync(HostSignal.BeforeProcessStart, new DiagnoserActionParameters(processListener.Process, benchmarkCase, benchmarkId), cancellationToken).ConfigureAwait(); @@ -161,14 +158,19 @@ private static async ValueTask Execute(BenchmarkCase benchmarkCas } } - private static Process CreateProcess(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, string args, IResolver resolver) + private Process CreateProcess(BenchmarkCase benchmarkCase, ArtifactsPaths artifactsPaths, string args, IResolver resolver) { - WasmRuntime runtime = (WasmRuntime)benchmarkCase.GetRuntime(); + // Resolve the engine to its full path. jsvu installs v8 on PATH as a v8.cmd wrapper, and Process.Start with + // UseShellExecute=false (required here for output redirection) can't launch a bare command that resolves to + // a .cmd - but it can launch the full path. Falls back to the configured value when it can't be resolved. + string engine = ProcessHelper.TryResolveExecutableInPath(settings.JavaScriptEngine, out string? resolvedEngine) + ? resolvedEngine + : settings.JavaScriptEngine; var start = new ProcessStartInfo { - FileName = runtime.JavaScriptEngine, - Arguments = runtime.JavaScriptEngineArgumentFormatter(runtime, artifactsPaths, args), + FileName = engine, + Arguments = settings.JavaScriptEngineArgumentFormatter(settings, artifactsPaths, args), WorkingDirectory = Path.Combine(artifactsPaths.BinariesDirectoryPath, "wwwroot"), UseShellExecute = false, RedirectStandardOutput = true, @@ -182,11 +184,10 @@ private static Process CreateProcess(BenchmarkCase benchmarkCase, ArtifactsPaths return new Process() { StartInfo = start }; } - private static async ValueTask Execute(Process process, BenchmarkCase benchmarkCase, AsyncProcessOutputReader processOutputReader, + private async ValueTask Execute(Process process, BenchmarkCase benchmarkCase, AsyncProcessOutputReader processOutputReader, BenchmarkId benchmarkId, Loggers.ILogger logger, int launchIndex, IDiagnoser diagnoser, CompositeInProcessDiagnoser compositeInProcessDiagnoser, IpcListener ipcListener, CancellationToken cancellationToken) { - WasmRuntime wasmRuntime = (WasmRuntime)benchmarkCase.GetRuntime(); List results; List prefixedOutput; await using (new ProcessCleanupHelper(process, processOutputReader, logger).ConfigureAwait(false)) @@ -214,10 +215,10 @@ private static async ValueTask Execute(Process process, Benchmark #pragma warning disable CA2016 // Forward the 'CancellationToken' parameter to methods await broker.ProcessData(cancellationToken) .AsTask() - .WaitAsync(TimeSpan.FromMinutes(wasmRuntime.ProcessTimeoutMinutes)).ConfigureAwait(false); + .WaitAsync(TimeSpan.FromMinutes(settings.ProcessTimeoutMinutes)).ConfigureAwait(false); #pragma warning restore CA2016 // Forward the 'CancellationToken' parameter to methods - if (!process.WaitForExit(milliseconds: (int)ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) + if (!process.WaitForExit(milliseconds: (int) ExecuteParameters.ProcessExitTimeout.TotalMilliseconds)) { logger.WriteLineInfo($"// The benchmarking process did not quit within {ExecuteParameters.ProcessExitTimeout.TotalSeconds} seconds, it's going to get force killed now."); } @@ -226,7 +227,7 @@ await broker.ProcessData(cancellationToken) { // Preserve pre-async-refactor behavior: log a message and let ProcessCleanupHelper // force-kill the process tree rather than propagating the timeout to the caller. - logger.WriteLineInfo($"// The benchmarking process did not finish within {wasmRuntime.ProcessTimeoutMinutes} minutes, it's going to get force killed now."); + logger.WriteLineInfo($"// The benchmarking process did not finish within {settings.ProcessTimeoutMinutes} minutes, it's going to get force killed now."); } results = broker.Results; diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmIpcType.cs b/src/BenchmarkDotNet/Toolchains/Wasm/WasmIpcType.cs similarity index 93% rename from src/BenchmarkDotNet/Toolchains/MonoWasm/WasmIpcType.cs rename to src/BenchmarkDotNet/Toolchains/Wasm/WasmIpcType.cs index b857e96d32..7ccc106b59 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmIpcType.cs +++ b/src/BenchmarkDotNet/Toolchains/Wasm/WasmIpcType.cs @@ -1,4 +1,4 @@ -namespace BenchmarkDotNet.Toolchains.MonoWasm +namespace BenchmarkDotNet.Toolchains.Wasm { /// /// Specifies the IPC mechanism to use for WASM benchmarks. diff --git a/src/BenchmarkDotNet/Toolchains/Wasm/WasmSettings.cs b/src/BenchmarkDotNet/Toolchains/Wasm/WasmSettings.cs new file mode 100644 index 0000000000..9bfffc84bf --- /dev/null +++ b/src/BenchmarkDotNet/Toolchains/Wasm/WasmSettings.cs @@ -0,0 +1,71 @@ +using System.Collections.Generic; +using BenchmarkDotNet.ConsoleArguments; +using BenchmarkDotNet.Toolchains.DotNetCli; + +namespace BenchmarkDotNet.Toolchains.Wasm; + +public sealed record WasmSettings : DotNetCliSettings +{ + public static readonly WasmSettings Default = new(); + + /// Full path to a JavaScript engine used to run the benchmarks. + public string JavaScriptEngine { get; init; } = "v8"; + /// Arguments for the JavaScript engine. + public string JavaScriptEngineArguments { get; init; } = ""; + /// Maximum time in minutes to wait for a single benchmark process to finish before force killing it. Default is 10 minutes. + public int ProcessTimeoutMinutes { get; init; } = 10; + /// + /// Specifies the IPC mechanism to use. Default is which automatically detects the JavaScript engine capabilities. + /// + public WasmIpcType IpcType { get; init; } = WasmIpcType.Auto; + /// Allows to format or customize the arguments passed to the JavaScript engine. + public Func JavaScriptEngineArgumentFormatter { get; init; } = DefaultArgumentFormatter; + /// Optional custom template for the generated main.mjs file. + public FileInfo? MainJsTemplate { get; init; } + + public WasmSettings() { } + + internal WasmSettings(CommandLineOptions options) : base(options) + { + JavaScriptEngine = options.WasmJavaScriptEngine ?? JavaScriptEngine; + JavaScriptEngineArguments = options.WasmJavaScriptEngineArguments ?? JavaScriptEngineArguments; + ProcessTimeoutMinutes = options.WasmProcessTimeoutMinutes; + MainJsTemplate = options.WasmMainJsTemplate; + } + + /// + // JavaScriptEngineArgumentFormatter (a delegate) is intentionally not surfaced. + public override void FillSettings(IDictionary settings) + { + base.FillSettings(settings); + settings[nameof(JavaScriptEngine)] = JavaScriptEngine; + settings[nameof(JavaScriptEngineArguments)] = JavaScriptEngineArguments; + settings[nameof(ProcessTimeoutMinutes)] = ProcessTimeoutMinutes; + settings[nameof(IpcType)] = IpcType; + settings[nameof(MainJsTemplate)] = MainJsTemplate?.FullName; + } + + // FileInfo compares by reference; compare MainJsTemplate by value and chain into the base record. The Func compares + // by delegate equality (the default DefaultArgumentFormatter is a static method, so two defaults are equal). + public bool Equals(WasmSettings? other) + => other is not null + && base.Equals(other) + && JavaScriptEngine == other.JavaScriptEngine + && JavaScriptEngineArguments == other.JavaScriptEngineArguments + && ProcessTimeoutMinutes == other.ProcessTimeoutMinutes + && IpcType == other.IpcType + && JavaScriptEngineArgumentFormatter == other.JavaScriptEngineArgumentFormatter + && MainJsTemplate?.FullName == other.MainJsTemplate?.FullName; + + public override int GetHashCode() + => HashCode.Combine(base.GetHashCode(), JavaScriptEngine, JavaScriptEngineArguments, ProcessTimeoutMinutes, IpcType, JavaScriptEngineArgumentFormatter, MainJsTemplate?.FullName); + + private static string DefaultArgumentFormatter(WasmSettings settings, ArtifactsPaths artifactsPaths, string args) + { + return Path.GetFileNameWithoutExtension(settings.JavaScriptEngine).ToLower() switch + { + "node" or "bun" => $"{settings.JavaScriptEngineArguments} {artifactsPaths.ExecutablePath} -- --run {artifactsPaths.ProgramName}.dll {args}", + _ => $"{settings.JavaScriptEngineArguments} --module {artifactsPaths.ExecutablePath} -- --run {artifactsPaths.ProgramName}.dll {args}", + }; + } +} diff --git a/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs index 369265cb92..521d1618cb 100644 --- a/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs +++ b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs @@ -1,7 +1,7 @@ using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Extensions; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.DotNetCli; using System.ComponentModel; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; @@ -13,26 +13,25 @@ internal static class DotNetSdkValidator { private static readonly Lazy> cachedFrameworkSdks = new Lazy>(GetInstalledFrameworkSdks, true); - public static IEnumerable ValidateCoreSdks(string? customDotNetCliPath, BenchmarkCase benchmark) + public static IEnumerable ValidateCoreSdks(FileInfo? customDotNetCliPath, BenchmarkCase benchmark) { if (IsCliPathInvalid(customDotNetCliPath, benchmark, out ValidationError? cliPathError)) { yield return cliPathError; yield break; } - var requiredSdkVersion = benchmark.GetRuntime().RuntimeMoniker.GetRuntimeVersion(); - if (!GetInstalledDotNetSdks(customDotNetCliPath).Any(sdk => sdk >= requiredSdkVersion)) + var requiredSdkVersion = benchmark.GetRuntime().Version; + if (requiredSdkVersion != null && !GetInstalledDotNetSdks(customDotNetCliPath).Any(sdk => sdk >= requiredSdkVersion)) { - yield return new ValidationError(true, $"The required .NET Core SDK version {requiredSdkVersion} or higher for runtime moniker {benchmark.Job.Environment.Runtime!.RuntimeMoniker} is not installed.", benchmark); + yield return new ValidationError(true, $"The required .NET Core SDK version {requiredSdkVersion} or higher for runtime {benchmark.GetRuntime()} is not installed.", benchmark); } } public static IEnumerable ValidateFrameworkSdks(BenchmarkCase benchmark) { - var targetRuntime = benchmark.Job.Environment.HasValue(EnvironmentMode.RuntimeCharacteristic) - ? benchmark.Job.Environment.Runtime! - : ClrRuntime.GetTargetOrCurrentVersion(benchmark.Descriptor.Type.Assembly); - var requiredSdkVersion = targetRuntime.RuntimeMoniker.GetRuntimeVersion(); + var targetRuntime = benchmark.GetRuntime() as ClrRuntime + ?? ClrRuntime.GetTargetOrCurrentVersion(benchmark.Descriptor.Type.Assembly); + var requiredSdkVersion = targetRuntime.Version; var installedVersionString = cachedFrameworkSdks.Value.FirstOrDefault(); if (installedVersionString == null || Version.TryParse(installedVersionString, out var installedVersion) && installedVersion < requiredSdkVersion) @@ -41,11 +40,11 @@ public static IEnumerable ValidateFrameworkSdks(BenchmarkCase b } } - public static bool IsCliPathInvalid(string? customDotNetCliPath, BenchmarkCase benchmarkCase, [NotNullWhen(true)] out ValidationError? validationError) + public static bool IsCliPathInvalid(FileInfo? customDotNetCliPath, BenchmarkCase benchmarkCase, [NotNullWhen(true)] out ValidationError? validationError) { validationError = null; - if (string.IsNullOrEmpty(customDotNetCliPath) && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) + if (customDotNetCliPath is null && !HostEnvironmentInfo.GetCurrent().IsDotNetCliInstalled()) { validationError = new ValidationError(true, $"BenchmarkDotNet requires dotnet SDK to be installed or path to local dotnet cli provided in explicit way using `--cli` argument, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", @@ -54,7 +53,7 @@ public static bool IsCliPathInvalid(string? customDotNetCliPath, BenchmarkCase b return true; } - if (!string.IsNullOrEmpty(customDotNetCliPath) && !File.Exists(customDotNetCliPath)) + if (customDotNetCliPath is { Exists: false }) { validationError = new ValidationError(true, $"Provided custom dotnet cli path does not exist, benchmark '{benchmarkCase.DisplayInfo}' will not be executed", @@ -66,9 +65,9 @@ public static bool IsCliPathInvalid(string? customDotNetCliPath, BenchmarkCase b return false; } - private static IEnumerable GetInstalledDotNetSdks(string? customDotNetCliPath) + private static IEnumerable GetInstalledDotNetSdks(FileInfo? customDotNetCliPath) { - string dotnetExecutable = string.IsNullOrEmpty(customDotNetCliPath) ? "dotnet" : customDotNetCliPath!; + string dotnetExecutable = customDotNetCliPath?.FullName ?? DotNetCliCommandExecutor.DefaultDotNetCliPath.Value; var startInfo = new ProcessStartInfo(dotnetExecutable, "--list-sdks") { RedirectStandardOutput = true, diff --git a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs b/src/BenchmarkDotNet/Validators/RuntimeValidator.cs deleted file mode 100644 index dd25e7d74e..0000000000 --- a/src/BenchmarkDotNet/Validators/RuntimeValidator.cs +++ /dev/null @@ -1,40 +0,0 @@ -using BenchmarkDotNet.Characteristics; -using BenchmarkDotNet.Toolchains; - -namespace BenchmarkDotNet.Validators; - -/// -/// Validator for runtime characteristic. -/// -/// -public class RuntimeValidator : IValidator -{ - public static readonly IValidator DontFailOnError = new RuntimeValidator(); - - private RuntimeValidator() { } - - public bool TreatsWarningsAsErrors => false; - - public async IAsyncEnumerable ValidateAsync(ValidationParameters input) - { - var allBenchmarks = input.Benchmarks.ToArray(); - var nullRuntimeBenchmarks = allBenchmarks.Where(x => x.Job.Environment.Runtime == null).ToArray(); - - // There is no validation error if all the runtimes are set or if all the runtimes are null. - if (allBenchmarks.Length == nullRuntimeBenchmarks.Length) - { - yield break; - } - - foreach (var benchmark in nullRuntimeBenchmarks.Where(x => !x.GetToolchain().IsInProcess)) - { - var job = benchmark.Job; - var jobText = job.HasValue(CharacteristicObject.IdCharacteristic) - ? job.Id - : CharacteristicSetPresenter.Display.ToPresentation(job); // Use job text representation instead for auto generated JobId. - - var message = $"Job({jobText}) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; - yield return new ValidationError(false, message); - } - } -} diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/General/RuntimeAndToolchainAnalyzerTests.cs b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/General/RuntimeAndToolchainAnalyzerTests.cs new file mode 100644 index 0000000000..deb036e473 --- /dev/null +++ b/tests/BenchmarkDotNet.Analyzers.Tests/AnalyzerTests/General/RuntimeAndToolchainAnalyzerTests.cs @@ -0,0 +1,143 @@ +using BenchmarkDotNet.Analyzers.General; +using BenchmarkDotNet.Analyzers.Tests.Fixtures; + +namespace BenchmarkDotNet.Analyzers.Tests.AnalyzerTests.General; + +public class RuntimeAndToolchainAnalyzerTests +{ + public class RuntimeAndToolchainBothSet : AnalyzerTestFixture + { + public RuntimeAndToolchainBothSet() : base(RuntimeAndToolchainAnalyzer.RuntimeAndToolchainBothSetRule) { } + + [Fact] + public async Task Chain_runtime_then_toolchain_triggers_diagnostic() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => Job.Dry.{|#0:WithRuntime|}(CoreRuntime.Core80).WithToolchain(InProcessEmitToolchain.Default); + } + """; + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task Chain_toolchain_then_runtime_triggers_diagnostic() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => Job.Dry.WithToolchain(InProcessEmitToolchain.Default).{|#0:WithRuntime|}(CoreRuntime.Core80); + } + """; + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task Chain_runtime_only_does_not_trigger_diagnostic() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + + public class Config + { + public Job Setup() => Job.Dry.WithRuntime(CoreRuntime.Core80); + } + """; + + await RunAsync(); + } + + [Fact] + public async Task Chain_toolchain_only_does_not_trigger_diagnostic() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => Job.Dry.WithToolchain(InProcessEmitToolchain.Default); + } + """; + + await RunAsync(); + } + + [Fact] + public async Task Property_assignments_trigger_diagnostic_on_runtime() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + {|#0:job.Infrastructure.Runtime = CoreRuntime.Core80|}; + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """; + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task Object_initializer_triggers_diagnostic_on_runtime() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => new Job { Infrastructure = { {|#0:Runtime = CoreRuntime.Core80|}, Toolchain = InProcessEmitToolchain.Default } }; + } + """; + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task Property_runtime_only_does_not_trigger_diagnostic() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + + public class Config + { + public Job Setup() + { + var job = new Job(); + job.Infrastructure.Runtime = CoreRuntime.Core80; + return job; + } + } + """; + + await RunAsync(); + } + } +} diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/CodeFixTests/RuntimeAndToolchainCodeFixProviderTests.cs b/tests/BenchmarkDotNet.Analyzers.Tests/CodeFixTests/RuntimeAndToolchainCodeFixProviderTests.cs new file mode 100644 index 0000000000..fc9031e6f6 --- /dev/null +++ b/tests/BenchmarkDotNet.Analyzers.Tests/CodeFixTests/RuntimeAndToolchainCodeFixProviderTests.cs @@ -0,0 +1,573 @@ +using BenchmarkDotNet.Analyzers.General; +using BenchmarkDotNet.Analyzers.Tests.Fixtures; +using BenchmarkDotNet.CodeFixers; +using Microsoft.CodeAnalysis; + +namespace BenchmarkDotNet.Analyzers.Tests.CodeFixTests; + +public class RuntimeAndToolchainCodeFixProviderTests : CodeFixTestFixture +{ + public RuntimeAndToolchainCodeFixProviderTests() : base(RuntimeAndToolchainAnalyzer.RuntimeAndToolchainBothSetRule) { } + + [Fact] + public async Task CodeFix_removes_runtime_from_chain_runtime_then_toolchain() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => Job.Dry.{|#0:WithRuntime|}(CoreRuntime.Core80).WithToolchain(InProcessEmitToolchain.Default); + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => Job.Dry.WithToolchain(InProcessEmitToolchain.Default); + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_removes_runtime_from_chain_toolchain_then_runtime() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => Job.Dry.WithToolchain(InProcessEmitToolchain.Default).{|#0:WithRuntime|}(CoreRuntime.Core80); + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => Job.Dry.WithToolchain(InProcessEmitToolchain.Default); + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_removes_runtime_property_assignment_statement() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + {|#0:job.Infrastructure.Runtime = CoreRuntime.Core80|}; + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_removes_runtime_from_object_initializer() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => new Job { Infrastructure = { {|#0:Runtime = CoreRuntime.Core80|}, Toolchain = InProcessEmitToolchain.Default } }; + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => new Job { Infrastructure = { Toolchain = InProcessEmitToolchain.Default } }; + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_keeps_the_comment_above_the_removed_statement_without_reindenting() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + // pin the runtime + {|#0:job.Infrastructure.Runtime = CoreRuntime.Core80|}; + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + // pin the runtime + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_removes_a_top_level_statement() + { + // Under top-level statements the assignment is wrapped in a GlobalStatement; removing the inner statement + // instead of the wrapper leaves it empty, which the syntax remover throws on. + OutputKind = OutputKind.ConsoleApplication; + + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + var job = new Job(); + // pin the runtime + {|#0:job.Infrastructure.Runtime = CoreRuntime.Core80|}; + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + var job = new Job(); + // pin the runtime + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_is_not_offered_for_the_static_call_form() + { + // Dropping the link means replacing the invocation with its receiver, but here the receiver is the type, so + // doing that would discard the job argument and leave JobExtensions.WithToolchain(t) - which does not compile. + var code = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => JobExtensions.{|#0:WithRuntime|}(Job.Dry, CoreRuntime.Core80).WithToolchain(InProcessEmitToolchain.Default); + } + """.ReplaceLineEndings(); + + TestCode = code; + FixedCode = code; + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_is_not_offered_for_a_conditional_access_chain() + { + // The receiver of `job?.WithRuntime(r)` is not part of the invocation, so the link cannot be dropped by + // replacing the call with it. The diagnostic covers the whole invocation here rather than just the method + // name, because the analyzer's name lookup does not handle a member binding. + var code = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup(Job job) => job?{|#0:.WithRuntime(CoreRuntime.Core80)|}.WithToolchain(InProcessEmitToolchain.Default); + } + """.ReplaceLineEndings(); + + TestCode = code; + FixedCode = code; + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_is_not_offered_for_an_embedded_statement() + { + // Removing the body of a brace-less `if` would leave it without one, and the syntax remover throws rather than + // produce that, so no action is offered. The diagnostic still stands - FixedCode equals TestCode. + var code = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup(bool condition) + { + var job = new Job(); + if (condition) + {|#0:job.Infrastructure.Runtime = CoreRuntime.Core80|}; + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + TestCode = code; + FixedCode = code; + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_keeps_the_comment_above_an_initializer_element() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => new Job + { + Infrastructure = + { + // pin the runtime + {|#0:Runtime = CoreRuntime.Core80|}, + Toolchain = InProcessEmitToolchain.Default, + } + }; + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => new Job + { + Infrastructure = + { + // pin the runtime + Toolchain = InProcessEmitToolchain.Default, + } + }; + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_does_not_orphan_a_region_directive() + { + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + #region runtime + {|#0:job.Infrastructure.Runtime = CoreRuntime.Core80|}; + #endregion + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + #region runtime + #endregion + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_does_not_orphan_a_region_opened_inside_the_removed_statement() + { + // The #region sits in the leading trivia of an INTERIOR token of the statement, so it is inside the removed + // span - KeepLeadingTrivia covers only the first token's trivia. Without KeepUnbalancedDirectives the + // #endregion below is left orphaned and the fixed code does not compile (CS1028). + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + {|#0:job.Infrastructure.Runtime = + #region runtime + CoreRuntime.Core80|}; + #endregion + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + + #region runtime + #endregion + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_does_not_orphan_a_region_whose_endregion_is_inside_the_removed_statement() + { + // The mirror image: the #endregion is the one inside the removed span, leaving the #region above it + // unterminated (CS1038). + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + #region runtime + var job = new Job(); + {|#0:job.Infrastructure.Runtime = + #endregion + CoreRuntime.Core80|}; + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + #region runtime + var job = new Job(); + + #endregion + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_does_not_orphan_a_conditional_opened_inside_the_removed_statement() + { + // Same shape with #if/#endif. The condition has to be one that HOLDS for the analyzed compilation: DEBUG is + // not defined there, so an "#if DEBUG" region would be disabled text and the test input itself would not + // parse - the assignment would lose its right-hand side. + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + {|#0:job.Infrastructure.Runtime = + #if !DEBUG + CoreRuntime.Core80|}; + #endif + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() + { + var job = new Job(); + + #if !DEBUG + #endif + job.Infrastructure.Toolchain = InProcessEmitToolchain.Default; + return job; + } + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } + + [Fact] + public async Task CodeFix_does_not_orphan_a_conditional_opened_inside_a_removed_initializer_element() + { + // As above, the condition has to hold for the analyzed compilation: DEBUG is not defined there, so "#if + // DEBUG" would make the initializer disabled text and the test input itself would not parse. + TestCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => new Job + { + Infrastructure = + { + {|#0:Runtime = + #if !DEBUG + CoreRuntime.Core80|}, + #endif + Toolchain = InProcessEmitToolchain.Default, + } + }; + } + """.ReplaceLineEndings(); + + FixedCode = /* lang=c#-test */ """ + using BenchmarkDotNet.Environments; + using BenchmarkDotNet.Jobs; + using BenchmarkDotNet.Toolchains.InProcess.Emit; + + public class Config + { + public Job Setup() => new Job + { + Infrastructure = + { + + #if !DEBUG + #endif + Toolchain = InProcessEmitToolchain.Default, + } + }; + } + """.ReplaceLineEndings(); + + AddExpectedDiagnostic(0); + await RunAsync(); + } +} diff --git a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/CodeFixTestFixture.cs b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/CodeFixTestFixture.cs index bb9807624b..f97765ab92 100644 --- a/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/CodeFixTestFixture.cs +++ b/tests/BenchmarkDotNet.Analyzers.Tests/Fixtures/CodeFixTestFixture.cs @@ -70,6 +70,16 @@ protected string FixedCode set => _codeFixTest.FixedCode = value; } + /// Set to to compile the test code as top-level statements. + protected OutputKind OutputKind + { + set + { + _codeFixTest.TestState.OutputKind = value; + _codeFixTest.FixedState.OutputKind = value; + } + } + protected void AddExpectedDiagnostic(int markupKey, params object[] arguments) { if (_ruleUnderTest == null) diff --git a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs index b04e8d3640..bdc47e5e9c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.ManualRunning.MultipleFrameworks/MultipleFrameworksTest.cs @@ -1,6 +1,6 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; namespace BenchmarkDotNet.IntegrationTests.ManualRunning @@ -18,15 +18,15 @@ public MultipleFrameworksTest(ITestOutputHelper output) : base(output) [InlineData(RuntimeMoniker.Net48)] [InlineData(RuntimeMoniker.Net80)] [InlineData(RuntimeMoniker.Net10_0)] - public void EachFrameworkIsRebuilt(RuntimeMoniker runtime) + public void EachFrameworkIsRebuilt(string runtime) { - var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(runtime.GetRuntime()).WithEnvironmentVariable(TfmEnvVarName, runtime.ToString())); + var config = ManualConfig.CreateEmpty().AddJob(Job.Dry.WithRuntime(Runtime.Parse(runtime)).WithEnvironmentVariable(TfmEnvVarName, runtime)); CanExecute(config); } public class ValuePerTfm { - private const RuntimeMoniker moniker = + private const string moniker = #if NET462 RuntimeMoniker.Net462; #elif NET48 @@ -36,13 +36,13 @@ public class ValuePerTfm #elif NET10_0 RuntimeMoniker.Net10_0; #else - RuntimeMoniker.NotRecognized; + "NotRecognized"; #endif [Benchmark] public void ThrowWhenWrong() { - if (Environment.GetEnvironmentVariable(TfmEnvVarName) != moniker.ToString()) + if (Environment.GetEnvironmentVariable(TfmEnvVarName) != moniker) { throw new InvalidOperationException($"Has not been recompiled, the value was {moniker}, expected {Environment.GetEnvironmentVariable(TfmEnvVarName)}"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests.MonoBenchmarks/Benchmarks.cs b/tests/BenchmarkDotNet.IntegrationTests.MonoBenchmarks/Benchmarks.cs index b4cbae8cd2..65c32dfe7c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.MonoBenchmarks/Benchmarks.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.MonoBenchmarks/Benchmarks.cs @@ -18,7 +18,7 @@ public void Check() throw new Exception("This is not Mono runtime"); } - if (RuntimeInformation.GetCurrentRuntime() != MonoRuntime.Mono80) + if (RuntimeInformation.GetCurrentRuntime() != MonoCoreRuntime.Net80) { throw new Exception("Incorrect runtime detection"); } diff --git a/tests/BenchmarkDotNet.IntegrationTests.SharedDiagnosers/FinalizerBlockerDiagnoser.cs b/tests/BenchmarkDotNet.IntegrationTests.SharedDiagnosers/FinalizerBlockerDiagnoser.cs index 9d5744162f..730b081001 100644 --- a/tests/BenchmarkDotNet.IntegrationTests.SharedDiagnosers/FinalizerBlockerDiagnoser.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.SharedDiagnosers/FinalizerBlockerDiagnoser.cs @@ -2,11 +2,11 @@ using BenchmarkDotNet.Analysers; using BenchmarkDotNet.Diagnosers; using BenchmarkDotNet.Engines; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.MonoWasm; using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.IntegrationTests.SharedDiagnosers; @@ -25,10 +25,10 @@ public void DisplayResults(ILogger logger) { } public IEnumerable ProcessResults(DiagnoserResults results) => []; public ValueTask HandleAsync(HostSignal signal, DiagnoserActionParameters parameters, CancellationToken cancellationToken) => new(); public RunMode GetRunMode(BenchmarkCase benchmarkCase) - // Mono Wasm throws PlatformNotSupportedException from Monitor.Wait, and defers finalization to the JS event loop (single-threaded), + // Wasm throws PlatformNotSupportedException from Monitor.Wait, and defers finalization to the JS event loop (single-threaded), // so it's impossible for us to prevent the finalizer from running. The good thing is that means it cannot run during our synchronous // benchmarks, but it also means we should never add any async-yielding benchmark memory tests for Wasm. - => benchmarkCase.Job.Infrastructure.Toolchain is WasmToolchain + => benchmarkCase.GetRuntime() is WasmRuntime ? RunMode.None : RunMode.ExtraIteration; public InProcessDiagnoserHandlerData GetHandlerData(BenchmarkCase benchmarkCase) => new(typeof(FinalizerBlockerDiagnoserHandler), null); diff --git a/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs b/tests/BenchmarkDotNet.IntegrationTests.SharedDiagnosers/MockInProcessDiagnoser.cs similarity index 91% rename from tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs rename to tests/BenchmarkDotNet.IntegrationTests.SharedDiagnosers/MockInProcessDiagnoser.cs index 15ea047136..44f13872c1 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/Diagnosers/MockInProcessDiagnoser.cs +++ b/tests/BenchmarkDotNet.IntegrationTests.SharedDiagnosers/MockInProcessDiagnoser.cs @@ -7,6 +7,9 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Validators; +// Lives in the SharedDiagnosers project (not the main test project) so the in-process diagnoser handler is reachable +// when building benchmarks that live in their own projects (Mono, Wasm). The namespace is kept as-is so consumers +// don't need to change their usings. namespace BenchmarkDotNet.IntegrationTests.Diagnosers; public abstract class BaseMockInProcessDiagnoser(RunMode runMode) : IInProcessDiagnoser @@ -86,4 +89,4 @@ public ValueTask HandleAsync(BenchmarkSignal signal, InProcessDiagnoserActionArg // Diagnosers are made unique per-type rather than per-instance, so we have to create separate types to test multiple. public sealed class MockInProcessDiagnoser1(RunMode runMode) : BaseMockInProcessDiagnoser(runMode) { } public sealed class MockInProcessDiagnoser2(RunMode runMode) : BaseMockInProcessDiagnoser(runMode) { } -public sealed class MockInProcessDiagnoser3(RunMode runMode) : BaseMockInProcessDiagnoser(runMode) { } \ No newline at end of file +public sealed class MockInProcessDiagnoser3(RunMode runMode) : BaseMockInProcessDiagnoser(runMode) { } diff --git a/tests/BenchmarkDotNet.IntegrationTests.WasmBenchmarks/BenchmarkDotNet.IntegrationTests.WasmBenchmarks.csproj b/tests/BenchmarkDotNet.IntegrationTests.WasmBenchmarks/BenchmarkDotNet.IntegrationTests.WasmBenchmarks.csproj new file mode 100644 index 0000000000..7199e4f275 --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.WasmBenchmarks/BenchmarkDotNet.IntegrationTests.WasmBenchmarks.csproj @@ -0,0 +1,18 @@ + + + + BenchmarkDotNet.IntegrationTests.WasmBenchmarks + $(AssemblyTitle) + + net472;net10.0 + + + + + + + + diff --git a/tests/BenchmarkDotNet.IntegrationTests.WasmBenchmarks/Benchmarks.cs b/tests/BenchmarkDotNet.IntegrationTests.WasmBenchmarks/Benchmarks.cs new file mode 100644 index 0000000000..c1746e779c --- /dev/null +++ b/tests/BenchmarkDotNet.IntegrationTests.WasmBenchmarks/Benchmarks.cs @@ -0,0 +1,19 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Portability; + +namespace BenchmarkDotNet.IntegrationTests.WasmBenchmarks; + +// These benchmarks live in their own project so the WASM tests can build/AOT them without pulling the xunit test +// framework into the benchmark app (AOT-compiling the xunit runner assemblies fails). + +public class WasmBenchmark +{ + [Benchmark] + public void Check() + { + if (!RuntimeInformation.IsWasm) + { + throw new Exception("Incorrect runtime detection"); + } + } +} diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs index 566c155c03..63ccd02838 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncBenchmarksTests.cs @@ -45,10 +45,10 @@ public AsyncBenchmarksTests(ITestOutputHelper output) : base(output) { } public static IEnumerable GetToolchains() => [ - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = true }), - new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = true }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), + InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), Job.Default.GetToolchain() ]; diff --git a/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs b/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs index d443c43074..e90eebe53f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/AsyncEnumerableBenchmarksTests.cs @@ -11,18 +11,18 @@ public class AsyncEnumerableBenchmarksTests(ITestOutputHelper output) : Benchmar { public static TheoryData GetAllToolchains() => [ - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = true }), - new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = true }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), + InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), Job.Default.GetToolchain() ]; // InProcessNoEmitToolchain does not support custom async enumerables or [AsyncCallerType]. public static TheoryData GetCustomSupportedToolchains() => [ - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = true }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), Job.Default.GetToolchain() ]; diff --git a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj index 768cccaaa7..c497ebb9f1 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj +++ b/tests/BenchmarkDotNet.IntegrationTests/BenchmarkDotNet.IntegrationTests.csproj @@ -36,6 +36,7 @@ + diff --git a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs index 2df6240de8..cbc9236387 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/BuildTimeoutTests.cs @@ -27,11 +27,8 @@ public void WhenBuildTakesMoreTimeThanTheTimeoutTheBuildIsCancelled() var config = ManualConfig.CreateEmpty() .WithBuildTimeout(timeout) .AddJob(Job.Dry - .WithRuntime(NativeAotRuntime.Net10_0) - .WithToolchain(NativeAotToolchain.CreateBuilder() - .UseNuGet("10.0.0", "https://api.nuget.org/v3/index.json") - .TargetFrameworkMoniker("net10.0") - .ToToolchain())); + .WithToolchain(CsProjNativeAotToolchain.From(NativeAotRuntime.Net80, + new NativeAotSettings().WithNuGet("8.0.0", "https://api.nuget.org/v3/index.json")))); var summary = CanExecute(config, fullValidation: false); diff --git a/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs index fe0b3940cd..3130471618 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CallerThreadTests.cs @@ -21,8 +21,8 @@ public class CallerThreadTests(ITestOutputHelper output) : BenchmarkTestExecutor { public static TheoryData GetToolchains() => [ - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = false }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), Job.Default.GetToolchain() ]; diff --git a/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs index 8e37f112dd..31fc65091c 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs @@ -11,11 +11,9 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; -using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.InProcess.NoEmit; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.Wasm; using BenchmarkDotNet.Validators; namespace BenchmarkDotNet.IntegrationTests; @@ -57,15 +55,13 @@ public void BenchmarkWithCancellationTokenProperty_ReceivesToken_InProcessEmit() [InlineData("node")] public void BenchmarkWithCancellationTokenProperty_ReceivesToken_Wasm(string javaScriptEngine) { - var dotnetVersion = "net10.0"; var logger = new OutputLogger(Output); - var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, runtimeFrameworkVersion: null!, "Wasm", aotCompilerMode: MonoAotCompilerMode.mini); + var wasmSettings = new WasmSettings { JavaScriptEngine = javaScriptEngine }; var config = ManualConfig.CreateEmpty() .AddLogger(logger) .AddJob(Job.Dry - .WithRuntime(new WasmRuntime(dotnetVersion, RuntimeMoniker.WasmNet10_0, "wasm", false, javaScriptEngine)) - .WithToolchain(WasmToolchain.From(netCoreAppSettings))) + .WithToolchain(CsProjMonoWasmToolchain.From(MonoWasmRuntime.Net10_0, wasmSettings))) .WithBuildTimeout(TimeSpan.FromSeconds(240)) .WithOption(ConfigOptions.LogBuildOutput, true) .WithOption(ConfigOptions.GenerateMSBuildBinLog, false); @@ -123,15 +119,13 @@ public async Task RunWithCancellationTokenIsCancelled_Wasm(string javaScriptEngi var cts = new CancellationTokenSource(); var diagnoser = new CancelAfterFirstIterationDiagnoser(cts); - var dotnetVersion = "net10.0"; var logger = new OutputLogger(Output); - var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, runtimeFrameworkVersion: null!, "Wasm", aotCompilerMode: MonoAotCompilerMode.mini); + var wasmSettings = new WasmSettings { JavaScriptEngine = javaScriptEngine }; var config = ManualConfig.CreateEmpty() .AddLogger(logger) .AddJob(Job.Dry - .WithRuntime(new WasmRuntime(dotnetVersion, RuntimeMoniker.WasmNet10_0, "wasm", false, javaScriptEngine)) - .WithToolchain(WasmToolchain.From(netCoreAppSettings))) + .WithToolchain(CsProjMonoWasmToolchain.From(MonoWasmRuntime.Net10_0, wasmSettings))) .AddDiagnoser(diagnoser) .WithBuildTimeout(TimeSpan.FromSeconds(240)) .WithOption(ConfigOptions.LogBuildOutput, true) diff --git a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs index 26720c6c41..7daddd5089 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/DisassemblyDiagnoserTests.cs @@ -12,8 +12,9 @@ using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.Framework; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.NetCoreApp; using System.Runtime.CompilerServices; namespace BenchmarkDotNet.IntegrationTests @@ -31,9 +32,9 @@ public static IEnumerable GetAllJits() if (RuntimeInformation.IsFullFramework) { - yield return [Jit.LegacyJit, Platform.X86, CsProjClassicNetToolchain.Net472]; // 32bit LegacyJit for desktop .NET - yield return [Jit.LegacyJit, Platform.X64, CsProjClassicNetToolchain.Net472]; // 64bit LegacyJit for desktop .NET - yield return [Jit.RyuJit, Platform.X64, CsProjClassicNetToolchain.Net472]; // RyuJit for desktop .NET + yield return [Jit.LegacyJit, Platform.X86, CsProjFrameworkToolchain.Net472]; // 32bit LegacyJit for desktop .NET + yield return [Jit.LegacyJit, Platform.X64, CsProjFrameworkToolchain.Net472]; // 64bit LegacyJit for desktop .NET + yield return [Jit.RyuJit, Platform.X64, CsProjFrameworkToolchain.Net472]; // RyuJit for desktop .NET } else if (RuntimeInformation.IsNetCore) { diff --git a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs index 00364868fc..b1e6d3738b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/EventProcessorTests.cs @@ -7,6 +7,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Results; using BenchmarkDotNet.Validators; @@ -144,7 +145,7 @@ public void WhenUsingEventProcessorWithUnsupportedBenchmark() [Fact] public void WhenUsingEventProcessorWithBuildFailures() { - var toolchain = new Toolchain("Build Failure", new AllFailsGenerator(), null!, null!); + var toolchain = new MockToolchain("Build Failure", UnknownRuntime.Instance, new AllFailsGenerator(), null!, null!); var events = RunBenchmarksAndRecordEvents([typeof(ClassA)], toolchain: toolchain); Assert.Equal(9, events.Count); @@ -213,7 +214,7 @@ public async IAsyncEnumerable ValidateAsync(ValidationParameter public class AllUnsupportedToolchain : Toolchain { - public AllUnsupportedToolchain() : base("AllUnsupported", null!, null!, null!) + public AllUnsupportedToolchain() : base("AllUnsupported", UnknownRuntime.Instance, null!, null!, null!) { } diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs index 7c6b7bdfb8..cdefb3aedb 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessEmitTest.cs @@ -10,7 +10,7 @@ using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains.InProcess.Emit; -using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Toolchains.Framework; using JetBrains.Annotations; namespace BenchmarkDotNet.IntegrationTests @@ -48,7 +48,7 @@ private IConfig CreateInProcessAndRoslynConfig(OutputLogger logger, bool consume .WithConsumeTasksSynchronously(consumeTasksSynchronously)) .AddJob( Job.Dry - .WithToolchain(new RoslynToolchain()) + .WithToolchain(RoslynFrameworkToolchain.Default) .WithInvocationCount(4) .WithUnrollFactor(4) .WithConsumeTasksSynchronously(consumeTasksSynchronously)) @@ -412,7 +412,7 @@ public void InProcessEmitRunsOnCallerThreadWhenConfigured() var config = new ManualConfig() .AddJob(Job.Dry - .WithToolchain(new InProcessEmitToolchain(new InProcessEmitSettings { ExecuteOnSeparateThread = false })) + .WithToolchain(InProcessEmitToolchain.From(new InProcessEmitSettings { ExecuteOnSeparateThread = false })) .WithInvocationCount(UnrollFactor) .WithUnrollFactor(UnrollFactor)) .AddLogger(Output != null ? new OutputLogger(Output) : ConsoleLogger.Default) diff --git a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs index 269d3fc8aa..d0cd8ecaed 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/InProcessTest.cs @@ -331,7 +331,7 @@ public void InProcessNoEmitRunsOnCallerThreadWhenConfigured() var config = new ManualConfig() .AddJob(Job.Dry - .WithToolchain(new InProcessNoEmitToolchain(new InProcessNoEmitSettings { ExecuteOnSeparateThread = false })) + .WithToolchain(InProcessNoEmitToolchain.From(new InProcessNoEmitSettings { ExecuteOnSeparateThread = false })) .WithInvocationCount(UnrollFactor) .WithUnrollFactor(UnrollFactor)) .AddLogger(Output != null ? new OutputLogger(Output) : ConsoleLogger.Default) @@ -360,7 +360,7 @@ public void BenchmarkActionFactoryTaskYieldSupported() var factory = new YieldAwaitableBenchmarkActionFactory(); var config = new ManualConfig() .AddJob(Job.Dry - .WithToolchain(new InProcessNoEmitToolchain(new InProcessNoEmitSettings { BenchmarkActionFactory = factory })) + .WithToolchain(InProcessNoEmitToolchain.From(new InProcessNoEmitSettings { BenchmarkActionFactory = factory })) .WithInvocationCount(UnrollFactor) .WithUnrollFactor(UnrollFactor)) .AddLogger(Output != null ? new OutputLogger(Output) : ConsoleLogger.Default) diff --git a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs index 212b73076d..18e22e9a42 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/JitRuntimeValidationTest.cs @@ -15,7 +15,8 @@ public JitRuntimeValidationTest(ITestOutputHelper output) : base(output) { } // private const string LegacyJitNotAvailableForMono = "// ERROR: LegacyJIT is requested but it is not available for Mono"; private const string RyuJitNotAvailable = "// ERROR: RyuJIT is requested but it is not available in current environment"; - private const string ToolchainSupportsOnlyRyuJit = "Currently dotnet cli toolchain supports only RyuJit"; + // The CsProj toolchains prefix this with their concrete type name (e.g. "CsProjCoreToolchain supports only RyuJit"). + private const string ToolchainSupportsOnlyRyuJit = "supports only RyuJit"; [TheoryEnvSpecific("CLR is a valid job only on Windows", EnvRequirement.WindowsOnly)] [InlineData(Jit.LegacyJit, Platform.X86, null)] diff --git a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs index 39374e9da6..bb6a4e8f13 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/LargeAddressAwareTest.cs @@ -35,7 +35,7 @@ public void BenchmarkCanAllocateMoreThan2Gb_Core() Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); - Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime)); + Assert.Equal(1, summary.Reports.Count(report => report.BenchmarkCase.GetRuntime() is CoreRuntime)); Assert.Contains(".NET 10.0", summary.AllRuntimes); } @@ -64,7 +64,7 @@ public void BenchmarkCanAllocateMoreThan2Gb_Framework() Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); Assert.True(summary.Reports.All(report => report.ExecuteResults.Any())); - Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime)); + Assert.Equal(jobCount, summary.Reports.Count(report => report.BenchmarkCase.GetRuntime() is ClrRuntime)); Assert.Contains(".NET Framework", summary.AllRuntimes); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs index d7ebecee71..5371dc8dd1 100755 --- a/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MemoryDiagnoserTests.cs @@ -16,11 +16,9 @@ using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.InProcess.Emit; using BenchmarkDotNet.Toolchains.Mono; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains.Wasm; using BenchmarkDotNet.Toolchains.NativeAot; using BenchmarkDotNet.Validators; using System.Reflection; @@ -62,7 +60,7 @@ public void MemoryDiagnoserIsAccurate(IToolchain toolchain) long objectAllocationOverhead = IntPtr.Size * 2; // pointer to method table + object header word long arraySizeOverhead = IntPtr.Size; // array length - if (toolchain is MonoToolchain) + if (toolchain.Runtime is MonoCoreRuntime) { objectAllocationOverhead += IntPtr.Size; } @@ -81,7 +79,7 @@ public void MemoryDiagnoserSupportsNativeAOT() if (OsDetector.IsMacOS()) return; // currently not supported - MemoryDiagnoserIsAccurate(NativeAotToolchain.Net10_0); + MemoryDiagnoserIsAccurate(CsProjNativeAotToolchain.Net10_0); } [FactEnvSpecific("We don't want to test MonoVM twice (.NET Framework and .NET Core), and it's not supported on Windows+Arm", @@ -95,7 +93,7 @@ public void MemoryDiagnoserSupportsModernMono() long arraySizeOverhead = IntPtr.Size; // array length objectAllocationOverhead += IntPtr.Size; // Mono has an extra word - AssertAllocations(MonoToolchain.Mono80, typeof(MonoBenchmarks.AccurateAllocations), new Dictionary + AssertAllocations(CsProjMonoCoreToolchain.Mono80, typeof(MonoBenchmarks.AccurateAllocations), new Dictionary { { nameof(MonoBenchmarks.AccurateAllocations.EightBytesArray), 8 + objectAllocationOverhead + arraySizeOverhead }, { nameof(MonoBenchmarks.AccurateAllocations.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead }, @@ -103,30 +101,21 @@ public void MemoryDiagnoserSupportsModernMono() }); } - [TheoryEnvSpecific("We don't want to test Wasm twice (.NET Framework and .NET Core), and JSVU does not support ARM on Windows or Linux", - [EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR])] - [InlineData(MonoAotCompilerMode.mini)] - // BUG: https://github.com/dotnet/BenchmarkDotNet/issues/3036 - [InlineData(MonoAotCompilerMode.wasm, Skip = "AOT is broken")] - public void MemoryDiagnoserSupportsMonoWasm(MonoAotCompilerMode aotCompilerMode) + [FactEnvSpecific("We don't want to test Wasm twice (.NET Framework and .NET Core), and JSVU does not support ARM on Windows or Linux", + EnvRequirement.DotNetCoreOnly, EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm, EnvRequirement.NonGitHubDraftPR)] + public void MemoryDiagnoserSupportsMonoWasm() { var ptrSize = sizeof(Int32); // We can't rely on IntPtr.Size, since we run on a different platform. Wasm is currently 32bit. var objectAllocationOverhead = ptrSize * 2; // pointer to method table + object header word var arraySizeOverhead = ptrSize * 2; // bounds + max_length var intTaskSize = 40; // We can't use CalculateRequiredSpace for AllocateTask since it calculates the size with IntPtr.Size. - var netCoreAppSettings = new NetCoreAppSettings("net10.0", runtimeFrameworkVersion: null!, "Wasm", aotCompilerMode: aotCompilerMode); - - var runtime = new WasmRuntime( - netCoreAppSettings.TargetFrameworkMoniker, RuntimeMoniker.WasmNet10_0, - "Wasm", aotCompilerMode == MonoAotCompilerMode.wasm, "v8"); - - AssertAllocations(WasmToolchain.From(netCoreAppSettings), typeof(AccurateAllocations), new Dictionary + AssertAllocations(CsProjMonoWasmToolchain.Net10_0, typeof(AccurateAllocations), new Dictionary { { nameof(AccurateAllocations.EightBytesArray), 8 + objectAllocationOverhead + arraySizeOverhead }, { nameof(AccurateAllocations.SixtyFourBytesArray), 64 + objectAllocationOverhead + arraySizeOverhead }, { nameof(AccurateAllocations.AllocateTask), intTaskSize }, - }, runtime: runtime); + }); } public class AllocatingGlobalSetupAndCleanup @@ -365,9 +354,9 @@ public void MemoryDiagnoserIsAccurateForMultiThreadedBenchmarks(IToolchain toolc }); } - private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, int iterationCount = 1, Runtime? runtime = null) + private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Dictionary benchmarksAllocationsValidators, int iterationCount = 1) { - var config = CreateConfig(toolchain, runtime, iterationCount); + var config = CreateConfig(toolchain, iterationCount); var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType, config); var summary = BenchmarkRunner.Run(benchmarks); @@ -403,7 +392,7 @@ private void AssertAllocations(IToolchain toolchain, Type benchmarkType, Diction } } - private IConfig CreateConfig(IToolchain toolchain, Runtime? runtime, + private IConfig CreateConfig(IToolchain toolchain, // Single iteration is enough for most of the tests. int iterationCount = 1) { @@ -417,11 +406,6 @@ private IConfig CreateConfig(IToolchain toolchain, Runtime? runtime, .WithJitTieringMode(JitTieringMode.Force) .WithToolchain(toolchain); - if (runtime is not null) - { - job = job.WithRuntime(runtime); - } - return ManualConfig.CreateEmpty() .AddJob(job) .WithBuildTimeout(TimeSpan.FromSeconds(480)) // Increase timeout for `MemoryDiagnoserSupportsModernMono` test on macos(x64) diff --git a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs index d8a1ae0352..3ed851d58b 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MonoTests.cs @@ -16,7 +16,7 @@ public void Mono80IsSupported() var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() .AddLogger(logger) - .AddJob(Job.Dry.WithRuntime(MonoRuntime.Mono80)) + .AddJob(Job.Dry.WithRuntime(MonoCoreRuntime.Net80)) .WithBuildTimeout(TimeSpan.FromSeconds(240)); // MonoBenchmark lives in a separate project that targets net8.0, because Mono packages // are no longer published for net9.0+ and this project no longer targets net8.0. diff --git a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs index 4b1c9e857e..0627531f13 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/MultipleRuntimesTest.cs @@ -37,12 +37,12 @@ public void SingleBenchmarkCanBeExecutedForMultipleRuntimes() Assert.True(summary.Reports.All(report => report.AllMeasurements.Any())); Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is ClrRuntime) + .Single(report => report.BenchmarkCase.GetRuntime() is ClrRuntime) .ExecuteResults .Any()); Assert.True(summary.Reports - .Single(report => report.BenchmarkCase.Job.Environment.Runtime is CoreRuntime) + .Single(report => report.BenchmarkCase.GetRuntime() is CoreRuntime) .ExecuteResults .Any()); @@ -57,7 +57,7 @@ public class C [Benchmark] public void B() { - Console.WriteLine($"// {RuntimeInformation.GetCurrentRuntime().GetToolchain()}"); + Console.WriteLine($"// {Job.Default.GetToolchain()}"); } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs index b275220872..74d3718af8 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/NativeAotTests.cs @@ -24,11 +24,12 @@ private bool IsAvx2Supported() private ManualConfig GetConfig() { - var toolchain = NativeAotToolchain.CreateBuilder().UseNuGet().IlcInstructionSet(IsAvx2Supported() ? "avx2" : "").ToToolchain(); + // we test against latest version for current TFM to make sure we avoid issues like #1055 + var toolchain = CsProjNativeAotToolchain.From(NativeAotRuntime.GetCurrentVersion(), + new NativeAotSettings { InstructionSet = IsAvx2Supported() ? "avx2" : "" }); return ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(NativeAotRuntime.GetCurrentVersion()) // we test against latest version for current TFM to make sure we avoid issues like #1055 .WithToolchain(toolchain) .WithEnvironmentVariable(NativeAotBenchmark.EnvVarKey, IsAvx2Supported().ToString().ToLower())); } diff --git a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs index d66f593577..16139d0fcb 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/R2RTests.cs @@ -1,6 +1,5 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Xunit; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Tests.XUnit; @@ -19,8 +18,7 @@ public void R2RToolchainCanExecuteBenchmarks() { var config = ManualConfig.CreateEmpty() .AddJob(Job.Dry - .WithRuntime(R2RRuntime.Net10_0) - .WithToolchain(R2RToolchain.NetCoreApp10_0)) + .WithToolchain(CsProjR2RToolchain.R2R10_0)) .WithBuildTimeout(TimeSpan.FromSeconds(360)); var summary = CanExecute(config); diff --git a/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs b/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs index 6ee7743a68..4d0d2c08d6 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/RoslynToolchainTest.cs @@ -1,7 +1,7 @@ using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Tests.XUnit; -using BenchmarkDotNet.Toolchains.Roslyn; +using BenchmarkDotNet.Toolchains.Framework; using System.Globalization; namespace BenchmarkDotNet.IntegrationTests @@ -28,7 +28,7 @@ public void CanExecuteWithNonDefaultUiCulture(string culture) CultureInfo.CurrentCulture = overrideCulture; CultureInfo.CurrentUICulture = overrideCulture; - var miniJob = Job.Dry.WithToolchain(RoslynToolchain.Instance); + var miniJob = Job.Dry.WithToolchain(RoslynFrameworkToolchain.Default); var config = CreateSimpleConfig(job: miniJob); CanExecute(config); diff --git a/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs b/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs index 5a1405bb78..89e943f4ae 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/RunAsyncTests.cs @@ -11,10 +11,10 @@ public class RunAsyncTests(ITestOutputHelper output) : BenchmarkTestExecutor(out { public static TheoryData GetToolchains() => [ - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessEmitToolchain(new() { ExecuteOnSeparateThread = true }), - new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = false }), - new InProcessNoEmitToolchain(new() { ExecuteOnSeparateThread = true }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), + InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = false }), + InProcessNoEmitToolchain.From(new() { ExecuteOnSeparateThread = true }), Job.Default.GetToolchain() ]; diff --git a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs index 2febff5c9e..3812a0c034 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ThreadingDiagnoserTests.cs @@ -35,7 +35,7 @@ public static IEnumerable GetToolchains() if (!ContinuousIntegration.IsGitHubActionsOnWindows() // no native dependencies && !OsDetector.IsMacOS()) // currently not supported { - yield return new object[] { NativeAotToolchain.Net10_0 }; + yield return new object[] { CsProjNativeAotToolchain.Net10_0 }; } } diff --git a/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs b/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs index 5975faf83b..6db925dbe4 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/ToolchainTest.cs @@ -4,12 +4,18 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Loggers; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.Parameters; using BenchmarkDotNet.Toolchains.Results; namespace BenchmarkDotNet.IntegrationTests { + public sealed class MockToolchain(string name, Runtime runtime, IGenerator generator, IBuilder builder, IExecutor executor) + : Toolchain(name, runtime, generator, builder, executor) + { + } + public class ToolchainTest(ITestOutputHelper output) : BenchmarkTestExecutor(output) { private class MyGenerator : IGenerator @@ -64,7 +70,7 @@ public void CustomToolchainsAreSupported() var generator = new MyGenerator(); var builder = new MyBuilder(); var executor = new MyExecutor(); - var myToolchain = new Toolchain("My", generator, builder, executor); + var myToolchain = new MockToolchain("My", UnknownRuntime.Instance, generator, builder, executor); var job = new Job(Job.Dry) { Infrastructure = { Toolchain = myToolchain } }; var config = CreateSimpleConfig(logger).AddJob(job); diff --git a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs index 940eeb55ea..78ff33566f 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WasmTests.cs @@ -1,14 +1,12 @@ -using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.IntegrationTests.Diagnosers; +using BenchmarkDotNet.IntegrationTests.WasmBenchmarks; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Portability; using BenchmarkDotNet.Tests.Loggers; using BenchmarkDotNet.Tests.XUnit; -using BenchmarkDotNet.Toolchains.DotNetCli; -using BenchmarkDotNet.Toolchains.MonoAotLLVM; -using BenchmarkDotNet.Toolchains.MonoWasm; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Wasm; namespace BenchmarkDotNet.IntegrationTests { @@ -22,30 +20,32 @@ namespace BenchmarkDotNet.IntegrationTests public class WasmTests(ITestOutputHelper output) : BenchmarkTestExecutor(output) { private const string V8SkipReason = "JSVU does not support ARM on Windows or Linux"; + // WASM AOT does not build on Windows arm64: the mono-aot-cross toolchain runs but does not emit the + // *_compiled_methods.txt token file, so WasmApp.Common.targets fails. It works on Linux arm64. + private const string WasmAotWindowsArmSkipReason = "WASM AOT does not build on Windows arm64"; [TheoryEnvSpecific(EnvRequirement.NonGitHubDraftPR)] - [InlineDataEnvSpecific([MonoAotCompilerMode.mini, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] - [InlineData(MonoAotCompilerMode.mini, "node")] - // BUG: https://github.com/dotnet/BenchmarkDotNet/issues/3036 - [InlineData(MonoAotCompilerMode.wasm, "v8", Skip = "AOT is broken")] - [InlineData(MonoAotCompilerMode.wasm, "node", Skip = "AOT is broken")] - public void WasmIsSupported(MonoAotCompilerMode aotCompilerMode, string javaScriptEngine) + [InlineDataEnvSpecific([RuntimeMoniker.MonoWasm10_0, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] + [InlineData(RuntimeMoniker.MonoWasm10_0, "node")] + [InlineDataEnvSpecific([RuntimeMoniker.MonoWasmAot10_0, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] + [InlineDataEnvSpecific([RuntimeMoniker.MonoWasmAot10_0, "node"], WasmAotWindowsArmSkipReason, [EnvRequirement.NonWindowsArm])] + // CoreWasm is not tested yet because it is still experimental . + //[InlineDataEnvSpecific([RuntimeMoniker.CoreWasm11_0, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] + //[InlineData(RuntimeMoniker.CoreWasm11_0, "node")] + public void WasmIsSupported(string runtimeMoniker, string javaScriptEngine) { - CanExecute(GetConfig(aotCompilerMode, javaScriptEngine)); + CanExecute(GetConfig(runtimeMoniker, javaScriptEngine)); } [TheoryEnvSpecific(EnvRequirement.NonGitHubDraftPR)] - [InlineDataEnvSpecific([MonoAotCompilerMode.mini, "v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] - [InlineData(MonoAotCompilerMode.mini, "node")] - // BUG: https://github.com/dotnet/BenchmarkDotNet/issues/3036 - [InlineData(MonoAotCompilerMode.wasm, "v8", Skip = "AOT is broken")] - [InlineData(MonoAotCompilerMode.wasm, "node", Skip = "AOT is broken")] - public void WasmSupportsInProcessDiagnosers(MonoAotCompilerMode aotCompilerMode, string javaScriptEngine) + [InlineDataEnvSpecific(["v8"], V8SkipReason, [EnvRequirement.NonWindowsArm, EnvRequirement.NonLinuxArm])] + [InlineData("node")] + public void WasmSupportsInProcessDiagnosers(string javaScriptEngine) { try { var diagnoser = new MockInProcessDiagnoser1(BenchmarkDotNet.Diagnosers.RunMode.NoOverhead); - var config = GetConfig(aotCompilerMode, javaScriptEngine).AddDiagnoser(diagnoser); + var config = GetConfig(RuntimeMoniker.MonoWasm10_0, javaScriptEngine).AddDiagnoser(diagnoser); CanExecute(config); @@ -64,38 +64,31 @@ public void WasmSupportsInProcessDiagnosers(MonoAotCompilerMode aotCompilerMode, public void WasmSupportsCustomMainJs(string javaScriptEngine, string customMainJs, WasmIpcType ipcType) { var mainJsTemplate = new FileInfo(Path.Combine("wwwroot", customMainJs)); - var summary = CanExecute(GetConfig(MonoAotCompilerMode.mini, javaScriptEngine, mainJsTemplate: mainJsTemplate, ipcType: ipcType)); + var summary = CanExecute(GetConfig(RuntimeMoniker.MonoWasm10_0, javaScriptEngine, mainJsTemplate: mainJsTemplate, ipcType: ipcType)); var standardOutput = summary.Reports.Single().ExecuteResults.Single().StandardOutput; Assert.Contains($"hello from {customMainJs}", standardOutput); } - private ManualConfig GetConfig(MonoAotCompilerMode aotCompilerMode, string javaScriptEngine, FileInfo? mainJsTemplate = null, WasmIpcType ipcType = WasmIpcType.Auto) + private ManualConfig GetConfig(string runtimeMoniker, string javaScriptEngine, FileInfo? mainJsTemplate = null, WasmIpcType ipcType = WasmIpcType.Auto) { - var dotnetVersion = "net10.0"; var logger = new OutputLogger(Output); - var netCoreAppSettings = new NetCoreAppSettings(dotnetVersion, runtimeFrameworkVersion: null!, "Wasm", aotCompilerMode: aotCompilerMode); + var wasmSettings = new WasmSettings { JavaScriptEngine = javaScriptEngine, MainJsTemplate = mainJsTemplate, IpcType = ipcType }; + IToolchain toolchain = Runtime.Parse(runtimeMoniker) switch + { + MonoWasmAotRuntime aot => CsProjMonoWasmAotToolchain.From(aot, wasmSettings), + MonoWasmRuntime mono => CsProjMonoWasmToolchain.From(mono, wasmSettings), + CoreWasmRuntime core => CsProjCoreWasmToolchain.From(core, wasmSettings), + var other => throw new ArgumentException($"'{runtimeMoniker}' is not a WASM runtime (parsed as {other.GetType().Name}).", nameof(runtimeMoniker)), + }; return ManualConfig.CreateEmpty() .AddLogger(logger) .AddJob(Job.Dry - .WithRuntime(new WasmRuntime(dotnetVersion, RuntimeMoniker.WasmNet10_0, "wasm", aotCompilerMode == MonoAotCompilerMode.wasm, javaScriptEngine, mainJsTemplate: mainJsTemplate, ipcType: ipcType)) - .WithToolchain(WasmToolchain.From(netCoreAppSettings))) + .WithToolchain(toolchain)) .WithBuildTimeout(TimeSpan.FromSeconds(480)) // Increase timeout for `WasmSupportsInProcessDiagnosers` test on macos(x64) .WithOption(ConfigOptions.LogBuildOutput, true) .WithOption(ConfigOptions.GenerateMSBuildBinLog, false); } - - public class WasmBenchmark - { - [Benchmark] - public void Check() - { - if (!RuntimeInformation.IsWasm) - { - throw new Exception("Incorrect runtime detection"); - } - } - } } } diff --git a/tests/BenchmarkDotNet.Tests/AppConfigGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/AppConfigGeneratorTests.cs index 067634e141..ca0c4a4b71 100644 --- a/tests/BenchmarkDotNet.Tests/AppConfigGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/AppConfigGeneratorTests.cs @@ -132,29 +132,6 @@ public async Task GeneratesRightJitSettings(Jit jit, string expectedRuntimeNode) AssertAreEqualIgnoringWhitespacesAndCase(customSettingsAndJit, destination.ToString()); } - [FactEnvSpecific("Full Framework is supported only on Windows", EnvRequirement.WindowsOnly)] - public async Task RemovesStartupSettingsForPrivateBuildsOfClr() - { - const string input = - "" + - "" + - "" + - ""; - - string withoutStartup = - "" + - "" + - $"{GcSettings}" + - "" + Environment.NewLine; - - using var source = new StringReader(input); - using var destination = new Utf8StringWriter(); - - await AppConfigGenerator.GenerateAsync(new Job { Environment = { Runtime = ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "4.0") } }.Freeze(), source, destination, Resolver, CancellationToken.None); - - AssertAreEqualIgnoringWhitespacesAndCase(withoutStartup, destination.ToString()); - } - [Fact] public async Task LeavsStartupSettingsIntactForNonPrivateBuildsOfClr() { @@ -174,7 +151,7 @@ public async Task LeavsStartupSettingsIntactForNonPrivateBuildsOfClr() using var source = new StringReader(input); using var destination = new Utf8StringWriter(); - await AppConfigGenerator.GenerateAsync(new Job { Environment = { Runtime = ClrRuntime.Net472 } }.Freeze(), source, destination, Resolver, CancellationToken.None); + await AppConfigGenerator.GenerateAsync(Job.Default.WithRuntime(ClrRuntime.Net472).Freeze(), source, destination, Resolver, CancellationToken.None); AssertAreEqualIgnoringWhitespacesAndCase(withoutStartup, destination.ToString()); } diff --git a/tests/BenchmarkDotNet.Tests/Columns/SettingsColumnTests.cs b/tests/BenchmarkDotNet.Tests/Columns/SettingsColumnTests.cs new file mode 100644 index 0000000000..c369865347 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Columns/SettingsColumnTests.cs @@ -0,0 +1,124 @@ +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Reports; +using BenchmarkDotNet.Tests.Mocks; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Toolchains.NetCoreApp; +using Xunit; + +namespace BenchmarkDotNet.Tests.Columns; + +public class SettingsColumnTests +{ + public class Bench + { + [Benchmark] public void Foo() { } + } + + [Fact] + public void SameToolchainWithDifferentSettingsShowsOnlyTheDifferingSetting() + { + // The two jobs use the same toolchain but different settings; job deduplication must keep them apart. + var config = ManualConfig.Create(DefaultConfig.Instance) + .AddJob(Job.Dry.WithToolchain(CsProjNativeAotToolchain.From(NativeAotRuntime.Net80, + new NativeAotSettings { OptimizationPreference = "Speed" }))) + .AddJob(Job.Dry.WithToolchain(CsProjNativeAotToolchain.From(NativeAotRuntime.Net80, + new NativeAotSettings { OptimizationPreference = "Size" }))); + + var columns = MockFactory.CreateSummary(typeof(Bench), config).GetColumns(); + + // The setting the two jobs disagree on is shown... + Assert.Contains(columns, c => c.ColumnName == nameof(NativeAotSettings.OptimizationPreference)); + // ...but a setting both jobs leave at the default is not. + Assert.DoesNotContain(columns, c => c.ColumnName == nameof(NativeAotSettings.InstructionSet)); + } + + [Fact] + public void DifferentToolchainsWithDefaultSettingsShowNoSettingColumns() + { + var config = ManualConfig.Create(DefaultConfig.Instance) + .AddJob(Job.Dry.WithToolchain(CsProjNativeAotToolchain.Net80)) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + + var columns = MockFactory.CreateSummary(typeof(Bench), config).GetColumns(); + + // A setting that only exists on one toolchain must not surface just because the other toolchain lacks it. + Assert.DoesNotContain(columns, c => c.Id.StartsWith("Settings.")); + } + + [Fact] + public void FileInfoSettingIsComparedByPathValueNotReference() + { + // Same CLI path passed as two distinct FileInfo instances; the jobs are kept apart by their differing + // runtime. The path column must stay hidden because the values compare equal by FullName, not by reference. + var config = ManualConfig.Create(DefaultConfig.Instance) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, new NetCoreAppSettings { CliPath = new FileInfo("dotnet") }))) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core90, new NetCoreAppSettings { CliPath = new FileInfo("dotnet") }))); + + var columns = MockFactory.CreateSummary(typeof(Bench), config).GetColumns(); + + Assert.DoesNotContain(columns, c => c.ColumnName == nameof(NetCoreAppSettings.CliPath)); + } + + [Fact] + public void FileInfoSettingWithDifferentPathsIsShown() + { + var config = ManualConfig.Create(DefaultConfig.Instance) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, new NetCoreAppSettings { CliPath = new FileInfo("dotnet-a") }))) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, new NetCoreAppSettings { CliPath = new FileInfo("dotnet-b") }))); + + var columns = MockFactory.CreateSummary(typeof(Bench), config).GetColumns(); + + Assert.Contains(columns, c => c.ColumnName == nameof(NetCoreAppSettings.CliPath)); + } + + [Fact] + public void SharedKeyAcrossSettingsTypesRendersInASingleColumn() + { + // CliPath comes from the shared DotNetCliSettings base, so both NativeAOT and Core expose it. Differing + // values across the two toolchains must surface as one column, not one per settings type. + var config = ManualConfig.Create(DefaultConfig.Instance) + .AddJob(Job.Dry.WithToolchain(CsProjNativeAotToolchain.From(NativeAotRuntime.Net80, new NativeAotSettings { CliPath = new FileInfo("dotnet-a") }))) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, new NetCoreAppSettings { CliPath = new FileInfo("dotnet-b") }))); + + var columns = MockFactory.CreateSummary(typeof(Bench), config).GetColumns(); + + Assert.Single(columns, c => c.ColumnName == nameof(NetCoreAppSettings.CliPath)); + } + + [Fact] + public void SettingMissingOnAToolchainRendersAsNotApplicable() + { + // RuntimeIdentifier is NativeAOT-only. The two NativeAOT jobs differ on it (so the column shows); the Core + // job has no such setting and must render as "NA", not "?". + var config = ManualConfig.Create(DefaultConfig.Instance) + .AddJob(Job.Dry.WithToolchain(CsProjNativeAotToolchain.From(NativeAotRuntime.Net80, new NativeAotSettings { RuntimeIdentifier = "win-x64" }))) + .AddJob(Job.Dry.WithToolchain(CsProjNativeAotToolchain.From(NativeAotRuntime.Net80, new NativeAotSettings { RuntimeIdentifier = "linux-x64" }))) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); + + var summary = MockFactory.CreateSummary(typeof(Bench), config); + var column = summary.GetColumns().Single(c => c.ColumnName == nameof(NativeAotSettings.RuntimeIdentifier)); + var coreCase = summary.BenchmarksCases.Single(bc => (bc.GetToolchain() as IHasSettings)?.Settings is NetCoreAppSettings); + + Assert.Equal("NA", column.GetValue(summary, coreCase)); + } + + [Fact] + public void NullSettingValueRendersAsQuestionMark() + { + // Two Core jobs: one sets CliPath, the other leaves it null. The column shows (they differ), and the unset + // one renders as "?" (the setting exists but is null), distinct from "NA". + var config = ManualConfig.Create(DefaultConfig.Instance) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, new NetCoreAppSettings { CliPath = new FileInfo("dotnet") }))) + .AddJob(Job.Dry.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core80, NetCoreAppSettings.Default))); + + var summary = MockFactory.CreateSummary(typeof(Bench), config); + var column = summary.GetColumns().Single(c => c.ColumnName == nameof(NetCoreAppSettings.CliPath)); + var nullCase = summary.BenchmarksCases.Single(bc => (bc.GetToolchain() as IHasSettings)?.Settings is NetCoreAppSettings { CliPath: null }); + + Assert.Equal("?", column.GetValue(summary, nullCase)); + } +} diff --git a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs index a735ff4d99..a768107092 100644 --- a/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs +++ b/tests/BenchmarkDotNet.Tests/ConfigParserTests.cs @@ -20,12 +20,15 @@ using BenchmarkDotNet.Tests.XUnit; using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Toolchains.CoreRun; -using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.DotNetCli; using BenchmarkDotNet.Toolchains.InProcess.Emit; +using BenchmarkDotNet.Toolchains.Mono; +using BenchmarkDotNet.Toolchains.Wasm; using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Toolchains.Framework; using Perfolizer.Horology; using System.Reflection; +using BenchmarkDotNet.Toolchains.NetCoreApp; namespace BenchmarkDotNet.Tests { @@ -260,11 +263,11 @@ public void CoreRunConfigParsedCorrectlyWhenRuntimeNotSpecified() Assert.Single(config.GetJobs()); CoreRunToolchain? toolchain = config.GetJobs().Single().GetToolchain() as CoreRunToolchain; Assert.NotNull(toolchain); - Assert.Equal(RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, - ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); // runtime was not specified so the current was used + Assert.Equal(((Runtime)RuntimeInformation.GetCurrentRuntime()).GetTfm(), + ((DotNetCliGenerator)toolchain.Generator).Settings.TargetFrameworkMoniker); // runtime was not specified so the current was used Assert.Equal(fakeCoreRunPath, toolchain.SourceCoreRun.FullName); - Assert.Equal(fakeDotnetCliPath, toolchain.CustomDotNetCliPath?.FullName); - Assert.Equal(fakeRestorePackages, toolchain.RestorePath?.FullName); + Assert.Equal(fakeDotnetCliPath, toolchain.Settings.CliPath?.FullName); + Assert.Equal(fakeRestorePackages, toolchain.Settings.PackagesPath?.FullName); } [FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.FullFrameworkOnly)] @@ -278,7 +281,7 @@ public void SpecifyingCoreRunWithFullFrameworkTargetsMostRecentTfm() CoreRunToolchain coreRunToolchain = (CoreRunToolchain)coreRunJob.GetToolchain(); DotNetCliGenerator generator = (DotNetCliGenerator)coreRunToolchain.Generator; - Assert.Equal("net11.0", generator.TargetFrameworkMoniker); + Assert.Equal("net11.0", generator.Settings.TargetFrameworkMoniker); } [FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)] @@ -300,16 +303,16 @@ public void SpecifyingCoreRunAndRuntimeCreatesTwoJobs() CoreRunToolchain coreRunToolchain = (CoreRunToolchain)coreRunJob.GetToolchain(); DotNetCliGenerator generator = (DotNetCliGenerator)coreRunToolchain.Generator; - Assert.Equal(RuntimeInformation.GetCurrentRuntime().MsBuildMoniker, generator.TargetFrameworkMoniker); + Assert.Equal(((Runtime)RuntimeInformation.GetCurrentRuntime()).GetTfm(), generator.Settings.TargetFrameworkMoniker); Assert.Equal(fakeCoreRunPath, coreRunToolchain.SourceCoreRun.FullName); - Assert.Equal(fakeDotnetCliPath, coreRunToolchain.CustomDotNetCliPath?.FullName); - Assert.Equal(fakeRestorePackages, coreRunToolchain.RestorePath?.FullName); + Assert.Equal(fakeDotnetCliPath, coreRunToolchain.Settings.CliPath?.FullName); + Assert.Equal(fakeRestorePackages, coreRunToolchain.Settings.PackagesPath?.FullName); CsProjCoreToolchain coreToolchain = (CsProjCoreToolchain)runtimeJob.GetToolchain(); generator = (DotNetCliGenerator)coreToolchain.Generator; - Assert.Equal(runtime, ((DotNetCliGenerator)coreToolchain.Generator).TargetFrameworkMoniker); - Assert.Equal(fakeDotnetCliPath, coreToolchain.CustomDotNetCliPath); - Assert.Equal(fakeRestorePackages, generator.PackagesPath); + Assert.Equal(runtime, ((DotNetCliGenerator)coreToolchain.Generator).Settings.TargetFrameworkMoniker); + Assert.Equal(fakeDotnetCliPath, coreToolchain.Settings.CliPath?.FullName); + Assert.Equal(fakeRestorePackages, generator.Settings.PackagesPath?.FullName); } [FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)] @@ -324,7 +327,7 @@ public void FirstJobIsBaseline_RuntimesCoreRun() Assert.Equal(3, config.GetJobs().Count()); Job baselineJob = config.GetJobs().Single(job => job.Meta.Baseline == true); Assert.False(baselineJob.GetToolchain() is CoreRunToolchain); - Assert.Equal(runtime1, ((DotNetCliGenerator)baselineJob.GetToolchain().Generator).TargetFrameworkMoniker); + Assert.Equal(runtime1, ((DotNetCliGenerator)baselineJob.GetToolchain().Generator).Settings.TargetFrameworkMoniker); } [FactEnvSpecific("It's impossible to determine TFM for CoreRunToolchain if host process is not .NET (Core) process", EnvRequirement.DotNetCoreOnly)] @@ -364,19 +367,33 @@ public void MonoPathParsedCorrectly() var config = ConfigParser.Parse(["-r", "mono", "--monoPath", fakeMonoPath], new OutputLogger(Output)).config; Assert.NotNull(config); - Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime mono && mono.CustomPath == fakeMonoPath); + var toolchain = Assert.IsType(config.GetJobs().Single().GetToolchain()); + Assert.IsType(toolchain.Runtime); + Assert.Equal(fakeMonoPath, toolchain.Settings.MonoPath?.FullName); } - [FactEnvSpecific("Testing local builds of Full .NET Framework is supported only on Windows", EnvRequirement.WindowsOnly)] - public void ClrVersionParsedCorrectly() + [Fact] + public void MonoAotIsParsedAndMonoPathHonored() { - const string clrVersion = "secret"; - var config = ConfigParser.Parse(["--clrVersion", clrVersion], new OutputLogger(Output)).config; + var fakeMonoPath = typeof(object).Assembly.Location; + var config = ConfigParser.Parse(["-r", "monoaot", "--monoPath", fakeMonoPath], new OutputLogger(Output)).config; Assert.NotNull(config); - Assert.Single(config.GetJobs()); - Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clr && clr.Version == clrVersion); + var toolchain = Assert.IsType(config.GetJobs().Single().GetToolchain()); + Assert.IsType(toolchain.Runtime); + Assert.Equal(fakeMonoPath, toolchain.Settings.MonoPath?.FullName); + } + + [Theory] + [InlineData("monowasm8.0", typeof(CsProjMonoWasmToolchain))] + [InlineData("monowasmaot8.0", typeof(CsProjMonoWasmAotToolchain))] + [InlineData("corewasm11.0", typeof(CsProjCoreWasmToolchain))] + public void WasmRuntimesResolveToTheirToolchains(string moniker, Type expectedToolchain) + { + var config = ConfigParser.Parse(["-r", moniker, GetDummyWasmEngine()], new OutputLogger(Output)).config; + + Assert.NotNull(config); + Assert.IsType(expectedToolchain, config.GetJobs().Single().GetToolchain()); } [Fact] @@ -387,10 +404,9 @@ public void IlCompilerPathParsedCorrectly() Assert.NotNull(config); Assert.Single(config.GetJobs()); - NativeAotToolchain? toolchain = config.GetJobs().Single().GetToolchain() as NativeAotToolchain; + CsProjNativeAotToolchain? toolchain = config.GetJobs().Single().GetToolchain() as CsProjNativeAotToolchain; Assert.NotNull(toolchain); - Generator generator = (Generator)toolchain.Generator; - Assert.Equal(fakePath.FullName, generator.Feeds["local"]); + Assert.Equal(fakePath.FullName, ((NativeAotSettings)toolchain.Settings).LocalIlcPackages?.FullName); } [Theory] @@ -418,14 +434,14 @@ public void DotNetCliParsedCorrectly(string tfm, bool isCore) if (isCore) { Assert.True(toolchain is CsProjCoreToolchain); - Assert.Equal(fakeDotnetCliPath, ((CsProjCoreToolchain)toolchain).CustomDotNetCliPath); + Assert.Equal(fakeDotnetCliPath, ((CsProjCoreToolchain)toolchain).Settings.CliPath?.FullName); } else { - Assert.True(toolchain is CsProjClassicNetToolchain); - Assert.Equal(fakeDotnetCliPath, ((CsProjClassicNetToolchain)toolchain).CustomDotNetCliPath); + Assert.True(toolchain is CsProjFrameworkToolchain); + Assert.Equal(fakeDotnetCliPath, ((DotNetCliBuilder)toolchain.Builder).CustomDotNetCliPath?.FullName); } - Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).Settings.TargetFrameworkMoniker); } [Theory] @@ -492,7 +508,7 @@ public void PackagesPathParsedCorrectly() Assert.Single(config.GetJobs()); var toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; Assert.NotNull(toolchain); - Assert.Equal(fakeRestoreDirectory, ((DotNetCliGenerator)toolchain.Generator).PackagesPath); + Assert.Equal(fakeRestoreDirectory, ((DotNetCliGenerator)toolchain.Generator).Settings.PackagesPath?.FullName); } [Fact] @@ -539,40 +555,41 @@ public void WhenUserDoesNotSpecifyWakeLockTheDefaultValueIsUsed() } [Theory] - [InlineData("net461")] - [InlineData("net462")] - [InlineData("net47")] - [InlineData("net471")] - [InlineData("net472")] - [InlineData("net48")] - [InlineData("net481")] - public void NetFrameworkMonikerParsedCorrectly(string tfm) + [InlineData("net461", "4.6.1")] + [InlineData("net462", "4.6.2")] + [InlineData("net47", "4.7")] + [InlineData("net471", "4.7.1")] + [InlineData("net472", "4.7.2")] + [InlineData("net48", "4.8")] + [InlineData("net481", "4.8.1")] + public void NetFrameworkMonikerParsedCorrectly(string tfm, string expectedVersion) { var config = ConfigParser.Parse(["-r", tfm], new OutputLogger(Output)).config; Assert.NotNull(config); Assert.Single(config.GetJobs()); - CsProjClassicNetToolchain? toolchain = config.GetJobs().Single().GetToolchain() as CsProjClassicNetToolchain; - Assert.NotNull(toolchain); - Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + // A netfx moniker only sets the runtime; the default toolchain is auto-selected and becomes the faster + // Roslyn toolchain when the requested version matches the host, so assert on the runtime, not the toolchain. + var runtime = Assert.IsType(config.GetJobs().Single().GetRuntime()); + Assert.Equal(Version.Parse(expectedVersion), runtime.Version); } [Theory] - [InlineData("net50")] - [InlineData("net5.0")] - [InlineData("net60")] - [InlineData("net6.0")] - [InlineData("net70")] - [InlineData("net7.0")] - [InlineData("net80")] - [InlineData("net8.0")] - [InlineData("net90")] - [InlineData("net9.0")] - [InlineData("net10_0")] - [InlineData("net10.0")] - [InlineData("net11_0")] - [InlineData("net11.0")] - public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) + [InlineData("net50", "net5.0")] + [InlineData("net5.0", "net5.0")] + [InlineData("net60", "net6.0")] + [InlineData("net6.0", "net6.0")] + [InlineData("net70", "net7.0")] + [InlineData("net7.0", "net7.0")] + [InlineData("net80", "net8.0")] + [InlineData("net8.0", "net8.0")] + [InlineData("net90", "net9.0")] + [InlineData("net9.0", "net9.0")] + [InlineData("net10_0", "net10.0")] + [InlineData("net10.0", "net10.0")] + [InlineData("net11_0", "net11.0")] + [InlineData("net11.0", "net11.0")] + public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm, string expectedTfm) { var config = ConfigParser.Parse(["-r", tfm], new OutputLogger(Output)).config; @@ -580,7 +597,7 @@ public void NetMonikersAreRecognizedAsNetCoreMonikers(string tfm) Assert.Single(config.GetJobs()); var toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; Assert.NotNull(toolchain); - Assert.Equal(tfm, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + Assert.Equal(expectedTfm, ((DotNetCliGenerator)toolchain.Generator).Settings.TargetFrameworkMoniker); } [Theory] @@ -594,7 +611,7 @@ public void PlatformSpecificMonikersAreSupported(string msBuildMoniker) Assert.Single(config.GetJobs()); var toolchain = config.GetJobs().Single().GetToolchain() as CsProjCoreToolchain; Assert.NotNull(toolchain); - Assert.Equal(msBuildMoniker, ((DotNetCliGenerator)toolchain.Generator).TargetFrameworkMoniker); + Assert.Equal(msBuildMoniker, ((DotNetCliGenerator)toolchain.Generator).Settings.TargetFrameworkMoniker); } [Fact] @@ -605,20 +622,16 @@ public void CanCompareFewDifferentRuntimes() Assert.NotNull(config); Assert.True(config.GetJobs().First().Meta.Baseline); // when the user provides multiple runtimes the first one should be marked as baseline - Assert.Single(config.GetJobs(), job => job.Environment.Runtime is ClrRuntime clrRuntime && clrRuntime.MsBuildMoniker == "net462"); - Assert.Single(config.GetJobs(), job => job.Environment.Runtime is MonoRuntime); + Assert.Single(config.GetJobs(), job => job.GetRuntime() is ClrRuntime clrRuntime && clrRuntime.Version == new Version(4, 6, 2)); + Assert.Single(config.GetJobs(), job => job.GetRuntime() is MonoRuntime); Assert.Single(config.GetJobs(), job => - job.Environment.Runtime is CoreRuntime coreRuntime && coreRuntime.MsBuildMoniker == "netcoreapp2.0" && - coreRuntime.RuntimeMoniker == RuntimeMoniker.NetCoreApp20); + job.GetRuntime() is CoreRuntime coreRuntime && coreRuntime.Version == new Version(2, 0)); Assert.Single(config.GetJobs(), job => - job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net8.0" && - nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot80); + job.GetRuntime() is NativeAotRuntime nativeAot && nativeAot.Version == new Version(8, 0)); Assert.Single(config.GetJobs(), job => - job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net9.0" && - nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot90); + job.GetRuntime() is NativeAotRuntime nativeAot && nativeAot.Version == new Version(9, 0)); Assert.Single(config.GetJobs(), job => - job.Environment.Runtime is NativeAotRuntime nativeAot && nativeAot.MsBuildMoniker == "net10.0" && - nativeAot.RuntimeMoniker == RuntimeMoniker.NativeAot10_0); + job.GetRuntime() is NativeAotRuntime nativeAot && nativeAot.Version == new Version(10, 0)); } [Theory] @@ -847,28 +860,28 @@ public void UsersCanSpecifyConsumeTasksSynchronously() [Fact(Skip = "This should be handled somehow at CommandLineParser level. See https://github.com/commandlineparser/commandline/pull/892")] public void UserCanSpecifyWasmArgs() { - var parsedConfiguration = ConfigParser.Parse(["--runtimes", "wasmnet80", "--wasmArgs", "--expose_wasm --module", GetDummyWasmEngine()], new OutputLogger(Output)); + var parsedConfiguration = ConfigParser.Parse(["--runtimes", "monowasm80", "--wasmArgs", "--expose_wasm --module", GetDummyWasmEngine()], new OutputLogger(Output)); Assert.True(parsedConfiguration.isSuccess); Assert.NotNull(parsedConfiguration.config); var jobs = parsedConfiguration.config.GetJobs(); foreach (var job in parsedConfiguration.config.GetJobs()) { - var wasmRuntime = Assert.IsType(job.Environment.Runtime); - Assert.Equal(" --expose_wasm --module", wasmRuntime.JavaScriptEngineArguments); + var wasmToolchain = Assert.IsAssignableFrom(job.GetToolchain()); + Assert.Equal(" --expose_wasm --module", ((WasmSettings)wasmToolchain.Settings).JavaScriptEngineArguments); } } [Fact] public void UserCanSpecifyWasmArgsUsingEquals() { - var parsedConfiguration = ConfigParser.Parse(["--runtimes", "wasmnet80", "--wasmArgs=--expose_wasm --module", GetDummyWasmEngine()], new OutputLogger(Output)); + var parsedConfiguration = ConfigParser.Parse(["--runtimes", "monowasm80", "--wasmArgs=--expose_wasm --module", GetDummyWasmEngine()], new OutputLogger(Output)); Assert.True(parsedConfiguration.isSuccess); Assert.NotNull(parsedConfiguration.config); var jobs = parsedConfiguration.config.GetJobs(); foreach (var job in parsedConfiguration.config.GetJobs()) { - var wasmRuntime = Assert.IsType(job.Environment.Runtime); - Assert.Equal("--expose_wasm --module", wasmRuntime.JavaScriptEngineArguments); + var wasmToolchain = Assert.IsAssignableFrom(job.GetToolchain()); + Assert.Equal("--expose_wasm --module", ((WasmSettings)wasmToolchain.Settings).JavaScriptEngineArguments); } } @@ -878,7 +891,7 @@ public void UserCanSpecifyWasmArgsViaResponseFile() var tempResponseFile = Path.GetRandomFileName(); File.WriteAllLines(tempResponseFile, [ - "--runtimes wasmnet80", + "--runtimes monowasm80", "--wasmArgs \"--expose_wasm --module\"", GetDummyWasmEngine() ]); @@ -888,22 +901,22 @@ public void UserCanSpecifyWasmArgsViaResponseFile() var jobs = parsedConfiguration.config.GetJobs(); foreach (var job in parsedConfiguration.config.GetJobs()) { - var wasmRuntime = Assert.IsType(job.Environment.Runtime); + var wasmToolchain = Assert.IsAssignableFrom(job.GetToolchain()); // We may need change assertion to just "--expose_wasm --module" // if https://github.com/commandlineparser/commandline/pull/892 lands - Assert.Equal(" --expose_wasm --module", wasmRuntime.JavaScriptEngineArguments); + Assert.Equal(" --expose_wasm --module", ((WasmSettings)wasmToolchain.Settings).JavaScriptEngineArguments); } } [Fact] public void UserCanSpecifyWasmMainJsTemplate() { - var parsedConfiguration = ConfigParser.Parse(["--runtimes", "wasmnet80", "--wasmMainJsTemplate", "./dummyFile.js", GetDummyWasmEngine()], new OutputLogger(Output)); + var parsedConfiguration = ConfigParser.Parse(["--runtimes", "monowasm80", "--wasmMainJsTemplate", "./dummyFile.js", GetDummyWasmEngine()], new OutputLogger(Output)); Assert.True(parsedConfiguration.isSuccess); var job = parsedConfiguration.config!.GetJobs().Single(); - var runtime = Assert.IsType(job.Environment.Runtime); - Assert.Equal("dummyFile.js", runtime.MainJsTemplate?.Name); + var wasmToolchain = Assert.IsAssignableFrom(job.GetToolchain()); + Assert.Equal("dummyFile.js", ((WasmSettings)wasmToolchain.Settings).MainJsTemplate?.Name); } [Theory] diff --git a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs index ab78df1888..271c377d7b 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/ImmutableConfigTests.cs @@ -5,6 +5,7 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Exporters; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Order; using BenchmarkDotNet.Reports; @@ -253,8 +254,8 @@ public void WhenTwoConfigsAreAddedTheRegularJobsAreJustAdded() var runnableJobs = added.GetJobs(); Assert.Equal(2, runnableJobs.Count()); - Assert.Single(runnableJobs, job => job.Environment.Runtime is ClrRuntime); - Assert.Single(runnableJobs, job => job.Environment.Runtime is CoreRuntime); + Assert.Single(runnableJobs, job => job.GetRuntime() is ClrRuntime); + Assert.Single(runnableJobs, job => job.GetRuntime() is CoreRuntime); } } @@ -273,8 +274,8 @@ public void WhenTwoConfigsAreAddedTheMutatorJobsAreAppliedToAllOtherJobs() Assert.Equal(2, runnableJobs.Count()); Assert.All(runnableJobs, job => Assert.Equal(warmupCount, job.Run.WarmupCount)); - Assert.Single(runnableJobs, job => job.Environment.Runtime is ClrRuntime); - Assert.Single(runnableJobs, job => job.Environment.Runtime is CoreRuntime); + Assert.Single(runnableJobs, job => job.GetRuntime() is ClrRuntime); + Assert.Single(runnableJobs, job => job.GetRuntime() is CoreRuntime); } } diff --git a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs index 51862f96d4..bc6526683d 100644 --- a/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs +++ b/tests/BenchmarkDotNet.Tests/Configs/JobTests.cs @@ -2,7 +2,8 @@ using BenchmarkDotNet.Engines; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains; +using BenchmarkDotNet.Toolchains.Framework; using System.Reflection; namespace BenchmarkDotNet.Tests.Configs @@ -389,7 +390,7 @@ public static void Test06CharacteristicHacks() a = InfrastructureMode.ToolchainCharacteristic; // will not throw: - a[j] = CsProjClassicNetToolchain.Net472; + a[j] = CsProjFrameworkToolchain.Net472; a[j] = null; a[j] = Characteristic.EmptyValue; Assert.Throws(() => a[j] = new EnvironmentMode()); // not assignable; @@ -412,7 +413,7 @@ public static void MutatorAppliedToOtherJobOverwritesOnlyTheConfiguredSettings() Assert.True(copy.HasValue(RunMode.MaxIterationCountCharacteristic)); Assert.Equal(20, copy.Run.MaxIterationCount); Assert.False(jobBefore.HasValue(RunMode.MaxIterationCountCharacteristic)); - Assert.True(copy.Environment.Runtime is CoreRuntime); + Assert.True(copy.GetRuntime() is CoreRuntime); Assert.False(copy.Meta.IsMutator); // the job does not became a mutator itself, this config should not be copied } diff --git a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs index e9119a7594..b1821045c9 100644 --- a/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/CsProjGeneratorTests.cs @@ -6,6 +6,7 @@ using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.Mocks; using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.NetCoreApp; using JetBrains.Annotations; using System.Reflection; using System.Xml; @@ -58,7 +59,7 @@ public void ItsPossibleToCustomizeProjectSdkForNetCoreAppsBasedOnTheImportOfSdk( [AssertionMethod] private void AssertParsedSdkName(string csProjContent, string targetFrameworkMoniker, string expectedSdkValue, bool isNetCore) { - var sut = new CsProjGenerator(targetFrameworkMoniker, "", "", "", isNetCore); + var sut = new CsProjGenerator(new NetCoreAppSettings { TargetFrameworkMoniker = targetFrameworkMoniker }, isNetCore); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(csProjContent); @@ -84,7 +85,7 @@ public void UseWpfSettingGetsCopied() "; - var sut = new CsProjGenerator("netcoreapp3.1", "", "", "", true); + var sut = new CsProjGenerator(new NetCoreAppSettings { TargetFrameworkMoniker = "netcoreapp3.1" }, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(withUseWpfTrue); @@ -114,7 +115,7 @@ public void SettingsFromPropsFileImportedUsingAbsolutePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.1", "", "", "", true); + var sut = new CsProjGenerator(new NetCoreAppSettings { TargetFrameworkMoniker = "netcoreapp3.1" }, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingAbsolutePath); @@ -146,7 +147,7 @@ public void SettingsFromPropsFileImportedUsingRelativePathGetCopies() "; - var sut = new CsProjGenerator("netcoreapp3.1", "", "", "", true); + var sut = new CsProjGenerator(new NetCoreAppSettings { TargetFrameworkMoniker = "netcoreapp3.1" }, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(importingRelativePath); @@ -168,7 +169,7 @@ public void RuntimeHostConfigurationOptionIsCopied() {runtimeHostConfigurationOptionChunk} "; - var sut = new CsProjGenerator("netcoreapp3.1", "", "", "", true); + var sut = new CsProjGenerator(new NetCoreAppSettings { TargetFrameworkMoniker = "netcoreapp3.1" }, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(source); @@ -188,7 +189,7 @@ public void WarningsAsErrorsSettingGetsCopied() "; - var sut = new CsProjGenerator("netcoreapp3.1", "", "", "", true); + var sut = new CsProjGenerator(new NetCoreAppSettings { TargetFrameworkMoniker = "netcoreapp3.1" }, true); var xmlDoc = new XmlDocument(); xmlDoc.LoadXml(withWarningsAsErrors); @@ -215,7 +216,7 @@ public void TheDefaultFilePathShouldBeUsedWhenAnAssemblyLocationIsEmpty() var benchmarkCase = BenchmarkCase.Create(target, Job.Default, ParameterInstances.Empty, config); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, config.CreateImmutableConfig(), 999, new([])) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", "", "", "", true); + var projectGenerator = new SteamLoadedBuildPartition(new NetCoreAppSettings { TargetFrameworkMoniker = "netcoreapp3.1" }, true); string binariesPath = projectGenerator.ResolvePathForBinaries(new BuildPartition(benchmarks, new Resolver()), programName); string expectedPath = Path.Combine(Path.Combine(Directory.GetCurrentDirectory(), "BenchmarkDotNet.Bin"), programName); @@ -229,7 +230,7 @@ public void TestAssemblyFilePathIsUsedWhenTheAssemblyLocationIsNotEmpty() var target = new Descriptor(MockFactory.MockType, MockFactory.MockMethodInfo); var benchmarkCase = BenchmarkCase.Create(target, Job.Default, ParameterInstances.Empty, ManualConfig.CreateEmpty().CreateImmutableConfig()); var benchmarks = new[] { new BenchmarkBuildInfo(benchmarkCase, ManualConfig.CreateEmpty().CreateImmutableConfig(), 0, new([])) }; - var projectGenerator = new SteamLoadedBuildPartition("netcoreapp3.1", "", "", "", true); + var projectGenerator = new SteamLoadedBuildPartition(new NetCoreAppSettings { TargetFrameworkMoniker = "netcoreapp3.1" }, true); var buildPartition = new BuildPartition(benchmarks, new Resolver()); string binariesPath = projectGenerator.ResolvePathForBinaries(buildPartition, programName); @@ -272,8 +273,8 @@ internal string ResolvePathForBinaries(BuildPartition buildPartition, string pro return base.GetBuildArtifactsDirectoryPath(buildPartition, programName); } - public SteamLoadedBuildPartition(string targetFrameworkMoniker, string cliPath, string packagesPath, string runtimeFrameworkVersion, bool isNetCore) - : base(targetFrameworkMoniker, cliPath, packagesPath, runtimeFrameworkVersion, isNetCore) { } + public SteamLoadedBuildPartition(NetCoreAppSettings settings, bool isNetCore) + : base(settings, isNetCore) { } } } } diff --git a/tests/BenchmarkDotNet.Tests/FrameworkVersionHelperTests.cs b/tests/BenchmarkDotNet.Tests/FrameworkVersionHelperTests.cs index 71384e65de..3f4db125af 100644 --- a/tests/BenchmarkDotNet.Tests/FrameworkVersionHelperTests.cs +++ b/tests/BenchmarkDotNet.Tests/FrameworkVersionHelperTests.cs @@ -23,7 +23,7 @@ public class FrameworkVersionHelperTests [InlineData("4.8.9032.0", "4.8.1")] public void ServicingVersionsAreMappedToCorrespondingReleaseVersions(string servicingVersion, string expectedReleaseVersion) { - Assert.Equal(expectedReleaseVersion, FrameworkVersionHelper.MapToReleaseVersion(servicingVersion)); + Assert.Equal(new Version(expectedReleaseVersion), FrameworkVersionHelper.MapToReleaseVersion(servicingVersion)); } } } diff --git a/tests/BenchmarkDotNet.Tests/Helpers/UniqueIdGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/Helpers/UniqueIdGeneratorTests.cs index 6b4136ad32..c95bbc15b6 100644 --- a/tests/BenchmarkDotNet.Tests/Helpers/UniqueIdGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Helpers/UniqueIdGeneratorTests.cs @@ -5,7 +5,7 @@ using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.NetCoreApp; namespace BenchmarkDotNet.Tests.Helpers; @@ -53,9 +53,9 @@ public void VerifyGeneratedUIds_WithCustomJobId() // Assert uids.Should().BeEquivalentTo( [ - "1f1ec424-c811-820a-a4ad-4def359fbc26", - "1ba530cd-a780-8707-a8f7-d59d1a80ad9d", - "1aa03476-a73c-8c01-918f-d80de359b386", + "1195d076-7324-88e9-be38-f686096150d6", + "1a59b0ad-fe5b-8bc4-b56e-4c4611296d0a", + "17102bb5-975f-8d91-a2d3-3c0da3addea2", "18ba3702-d8a3-8666-9b61-48e7a90e784a", "15fa977d-8202-81f3-abf4-c6d0c9d5b18c", diff --git a/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs index e9c907c232..c8bc72d21f 100644 --- a/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs +++ b/tests/BenchmarkDotNet.Tests/Jobs/JobIdGeneratorTests.cs @@ -1,6 +1,6 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.NetCoreApp; namespace BenchmarkDotNet.Tests.Jobs; @@ -19,9 +19,8 @@ public void AutoGenerateJobId(string expectedId, Job job) public static TheoryData GetTheoryData() => new TheoryData() { - {"Job-OOTPKI", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80) }, - {"Job-QAODSR", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp90) }, - {"Job-KHMDUZ", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp80).WithRuntime(CoreRuntime.Core80) }, - {"Job-JMDAGQ", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) }, + {"Job-GUNERI", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp90) }, + {"Job-HQSRPM", Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) }, + {"Job-GVKUBM", Job.Default.WithRuntime(CoreRuntime.Core10_0) }, }; } diff --git a/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs index 274f17f32f..af848c54cb 100644 --- a/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs +++ b/tests/BenchmarkDotNet.Tests/Order/JobOrderTests.cs @@ -1,7 +1,7 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Toolchains; -using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.NetCoreApp; namespace BenchmarkDotNet.Tests.Order; @@ -14,13 +14,10 @@ public void TestJobOrders_ByJobId() Job[] jobs = [ Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) - .WithRuntime(CoreRuntime.Core80) .WithId("v1.4.1"), Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) - .WithRuntime(CoreRuntime.Core90) .WithId("v1.4.10"), Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) - .WithRuntime(CoreRuntime.Core10_0) .WithId("v1.4.2"), ]; @@ -52,28 +49,29 @@ public void TestJobOrders_ByJobId() public void TestJobOrders_ByRuntime() { // Arrange + // Deliberately not in sorted order: with sorted input a comparer that returned 0 for every pair would still + // pass, because OrderBy is stable. Job[] jobs = [ - Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp10_0) - .WithRuntime(CoreRuntime.Core80), - Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp90) - .WithRuntime(CoreRuntime.Core90), - Job.Dry.WithToolchain(CsProjCoreToolchain.NetCoreApp80) - .WithRuntime(CoreRuntime.Core10_0), + Job.Dry.WithRuntime(CoreRuntime.Core10_0), + Job.Dry.WithRuntime(CoreRuntime.Core80), + Job.Dry.WithRuntime(CoreRuntime.Core90), ]; // Act // Verify jobs are sorted by Runtime name order. + // ToString(), not Name: Name is the runtime family (".NET") and is identical for every version here, which + // would make the assertion hold no matter how the jobs were ordered. var results = jobs.OrderBy(d => d, JobComparer.Default) - .Select(x => x.Job.Environment.GetRuntime().Name) + .Select(x => x.GetRuntime().ToString()) .ToArray(); // Assert var expected = new[] { - CoreRuntime.Core80.Name, - CoreRuntime.Core90.Name, - CoreRuntime.Core10_0.Name + CoreRuntime.Core80.ToString(), + CoreRuntime.Core90.ToString(), + CoreRuntime.Core10_0.ToString() }; Assert.Equal(expected, results); } @@ -92,15 +90,15 @@ public void TestJobOrders_ByToolchain() // Act // Verify jobs are sorted by Toolchain name order. var results = jobs.OrderBy(d => d, JobComparer.Default) - .Select(x => x.Job.GetToolchain().Name) + .Select(x => x.Job.GetToolchain().ToString()) .ToArray(); // Assert var expected = new[] { - CsProjCoreToolchain.NetCoreApp80.Name, - CsProjCoreToolchain.NetCoreApp90.Name, - CsProjCoreToolchain.NetCoreApp10_0.Name, + CsProjCoreToolchain.NetCoreApp80.ToString(), + CsProjCoreToolchain.NetCoreApp90.ToString(), + CsProjCoreToolchain.NetCoreApp10_0.ToString(), }; Assert.Equal(expected, results); } diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs index 81d35e488d..726c693f15 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs +++ b/tests/BenchmarkDotNet.Tests/Perfonar/PerfonarTests.cs @@ -76,7 +76,7 @@ public Task PerfonarTableTest(string key) "default05", new PerfonarTable( Root().Add( Enumerable.Range(0, 20).Select(index => - Job((RuntimeMoniker)index, Jit.RyuJit, index).Add( + Job(RuntimeMoniker.Net70, Jit.RyuJit, index).Add( Benchmark("Foo", index * 10 + 1 + "ns", index * 10 + 2 + "ns", index * 10 + 3 + "ns"), Benchmark("Bar", index * 10 + 6 + "ns", index * 10 + 7 + "ns", index * 10 + 8 + "ns") )).ToArray()), @@ -145,11 +145,13 @@ public Task PerfonarTableTest(string key) ] }; - private static EntryInfo Job(RuntimeMoniker? runtime = null, Jit? jit = null, int? affinity = null) => new EntryInfo + private static EntryInfo Job(string? runtime = null, Jit? jit = null, int? affinity = null) => new() { - Job = new JobInfo + // The runtime lives under Infrastructure (mirroring Job.Infrastructure), not Environment. + Job = new BdnJob { - Environment = new BdnEnvironment { Runtime = runtime, Jit = jit, Affinity = affinity } + Environment = new BdnEnvironment { Jit = jit, Affinity = affinity }, + Infrastructure = runtime is null ? null : new BdnInfrastructure { Runtime = runtime } } }; diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt index c8759facf6..e7d92d7501 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default02.verified.txt @@ -25,7 +25,8 @@ .iterationIndex .job .job.environment - .job.environment.runtime + .job.infrastructure + .job.infrastructure.runtime .unit .value @@ -56,7 +57,8 @@ .iterationIndex = 0 .job = .job.environment = - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 10 [Entry1] @@ -86,7 +88,8 @@ .iterationIndex = 1 .job = .job.environment = - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 11 [Entry2] @@ -116,7 +119,8 @@ .iterationIndex = 2 .job = .job.environment = - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 12 [Entry3] @@ -146,7 +150,8 @@ .iterationIndex = 0 .job = .job.environment = - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 20 [Entry4] @@ -176,7 +181,8 @@ .iterationIndex = 1 .job = .job.environment = - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 21 [Entry5] @@ -206,7 +212,8 @@ .iterationIndex = 2 .job = .job.environment = - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 22 [Entry6] @@ -236,7 +243,8 @@ .iterationIndex = 0 .job = .job.environment = - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 30 [Entry7] @@ -266,7 +274,8 @@ .iterationIndex = 1 .job = .job.environment = - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 31 [Entry8] @@ -296,7 +305,8 @@ .iterationIndex = 2 .job = .job.environment = - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 32 [Entry9] @@ -326,7 +336,8 @@ .iterationIndex = 0 .job = .job.environment = - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 40 [Entry10] @@ -356,7 +367,8 @@ .iterationIndex = 1 .job = .job.environment = - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 41 [Entry11] @@ -386,6 +398,7 @@ .iterationIndex = 2 .job = .job.environment = - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt index af8395637c..237ff2b073 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default03.verified.txt @@ -26,7 +26,8 @@ .job .job.environment .job.environment.jit - .job.environment.runtime + .job.infrastructure + .job.infrastructure.runtime .unit .value @@ -58,7 +59,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 30 [Entry1] @@ -89,7 +91,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 31 [Entry2] @@ -120,7 +123,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 32 [Entry3] @@ -151,7 +155,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 40 [Entry4] @@ -182,7 +187,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 41 [Entry5] @@ -213,6 +219,7 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt index 1179e76cd5..6ab28c9c5f 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default04.verified.txt @@ -26,7 +26,8 @@ .job .job.environment .job.environment.jit - .job.environment.runtime + .job.infrastructure + .job.infrastructure.runtime .unit .value @@ -58,7 +59,8 @@ .job = .job.environment = .job.environment.jit = LegacyJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 10 [Entry1] @@ -89,7 +91,8 @@ .job = .job.environment = .job.environment.jit = LegacyJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 11 [Entry2] @@ -120,7 +123,8 @@ .job = .job.environment = .job.environment.jit = LegacyJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 12 [Entry3] @@ -151,7 +155,8 @@ .job = .job.environment = .job.environment.jit = LegacyJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 20 [Entry4] @@ -182,7 +187,8 @@ .job = .job.environment = .job.environment.jit = LegacyJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 21 [Entry5] @@ -213,7 +219,8 @@ .job = .job.environment = .job.environment.jit = LegacyJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net481 .unit = ns .value = 22 [Entry6] @@ -244,7 +251,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 30 [Entry7] @@ -275,7 +283,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 31 [Entry8] @@ -306,7 +315,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 32 [Entry9] @@ -337,7 +347,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 40 [Entry10] @@ -368,7 +379,8 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 41 [Entry11] @@ -399,6 +411,7 @@ .job = .job.environment = .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 42 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt index b42c81bdab..bb25793e6d 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarIndexTest_key=default05.verified.txt @@ -27,7 +27,8 @@ .job.environment .job.environment.affinity .job.environment.jit - .job.environment.runtime + .job.infrastructure + .job.infrastructure.runtime .unit .value @@ -60,7 +61,8 @@ .job.environment = .job.environment.affinity = 0 .job.environment.jit = RyuJit - .job.environment.runtime = HostProcess + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 1 [Entry1] @@ -92,7 +94,8 @@ .job.environment = .job.environment.affinity = 0 .job.environment.jit = RyuJit - .job.environment.runtime = HostProcess + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 2 [Entry2] @@ -124,7 +127,8 @@ .job.environment = .job.environment.affinity = 0 .job.environment.jit = RyuJit - .job.environment.runtime = HostProcess + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 3 [Entry3] @@ -156,7 +160,8 @@ .job.environment = .job.environment.affinity = 0 .job.environment.jit = RyuJit - .job.environment.runtime = HostProcess + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 6 [Entry4] @@ -188,7 +193,8 @@ .job.environment = .job.environment.affinity = 0 .job.environment.jit = RyuJit - .job.environment.runtime = HostProcess + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 7 [Entry5] @@ -220,7 +226,8 @@ .job.environment = .job.environment.affinity = 0 .job.environment.jit = RyuJit - .job.environment.runtime = HostProcess + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 8 [Entry6] @@ -252,7 +259,8 @@ .job.environment = .job.environment.affinity = 1 .job.environment.jit = RyuJit - .job.environment.runtime = NotRecognized + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 11 [Entry7] @@ -284,7 +292,8 @@ .job.environment = .job.environment.affinity = 1 .job.environment.jit = RyuJit - .job.environment.runtime = NotRecognized + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 12 [Entry8] @@ -316,7 +325,8 @@ .job.environment = .job.environment.affinity = 1 .job.environment.jit = RyuJit - .job.environment.runtime = NotRecognized + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 13 [Entry9] @@ -348,7 +358,8 @@ .job.environment = .job.environment.affinity = 1 .job.environment.jit = RyuJit - .job.environment.runtime = NotRecognized + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 16 [Entry10] @@ -380,7 +391,8 @@ .job.environment = .job.environment.affinity = 1 .job.environment.jit = RyuJit - .job.environment.runtime = NotRecognized + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 17 [Entry11] @@ -412,7 +424,8 @@ .job.environment = .job.environment.affinity = 1 .job.environment.jit = RyuJit - .job.environment.runtime = NotRecognized + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 18 [Entry12] @@ -444,7 +457,8 @@ .job.environment = .job.environment.affinity = 2 .job.environment.jit = RyuJit - .job.environment.runtime = Mono + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 21 [Entry13] @@ -476,7 +490,8 @@ .job.environment = .job.environment.affinity = 2 .job.environment.jit = RyuJit - .job.environment.runtime = Mono + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 22 [Entry14] @@ -508,7 +523,8 @@ .job.environment = .job.environment.affinity = 2 .job.environment.jit = RyuJit - .job.environment.runtime = Mono + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 23 [Entry15] @@ -540,7 +556,8 @@ .job.environment = .job.environment.affinity = 2 .job.environment.jit = RyuJit - .job.environment.runtime = Mono + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 26 [Entry16] @@ -572,7 +589,8 @@ .job.environment = .job.environment.affinity = 2 .job.environment.jit = RyuJit - .job.environment.runtime = Mono + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 27 [Entry17] @@ -604,7 +622,8 @@ .job.environment = .job.environment.affinity = 2 .job.environment.jit = RyuJit - .job.environment.runtime = Mono + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 28 [Entry18] @@ -636,7 +655,8 @@ .job.environment = .job.environment.affinity = 3 .job.environment.jit = RyuJit - .job.environment.runtime = Net461 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 31 [Entry19] @@ -668,7 +688,8 @@ .job.environment = .job.environment.affinity = 3 .job.environment.jit = RyuJit - .job.environment.runtime = Net461 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 32 [Entry20] @@ -700,7 +721,8 @@ .job.environment = .job.environment.affinity = 3 .job.environment.jit = RyuJit - .job.environment.runtime = Net461 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 33 [Entry21] @@ -732,7 +754,8 @@ .job.environment = .job.environment.affinity = 3 .job.environment.jit = RyuJit - .job.environment.runtime = Net461 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 36 [Entry22] @@ -764,7 +787,8 @@ .job.environment = .job.environment.affinity = 3 .job.environment.jit = RyuJit - .job.environment.runtime = Net461 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 37 [Entry23] @@ -796,7 +820,8 @@ .job.environment = .job.environment.affinity = 3 .job.environment.jit = RyuJit - .job.environment.runtime = Net461 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 38 [Entry24] @@ -828,7 +853,8 @@ .job.environment = .job.environment.affinity = 4 .job.environment.jit = RyuJit - .job.environment.runtime = Net462 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 41 [Entry25] @@ -860,7 +886,8 @@ .job.environment = .job.environment.affinity = 4 .job.environment.jit = RyuJit - .job.environment.runtime = Net462 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 42 [Entry26] @@ -892,7 +919,8 @@ .job.environment = .job.environment.affinity = 4 .job.environment.jit = RyuJit - .job.environment.runtime = Net462 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 43 [Entry27] @@ -924,7 +952,8 @@ .job.environment = .job.environment.affinity = 4 .job.environment.jit = RyuJit - .job.environment.runtime = Net462 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 46 [Entry28] @@ -956,7 +985,8 @@ .job.environment = .job.environment.affinity = 4 .job.environment.jit = RyuJit - .job.environment.runtime = Net462 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 47 [Entry29] @@ -988,7 +1018,8 @@ .job.environment = .job.environment.affinity = 4 .job.environment.jit = RyuJit - .job.environment.runtime = Net462 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 48 [Entry30] @@ -1020,7 +1051,8 @@ .job.environment = .job.environment.affinity = 5 .job.environment.jit = RyuJit - .job.environment.runtime = Net47 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 51 [Entry31] @@ -1052,7 +1084,8 @@ .job.environment = .job.environment.affinity = 5 .job.environment.jit = RyuJit - .job.environment.runtime = Net47 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 52 [Entry32] @@ -1084,7 +1117,8 @@ .job.environment = .job.environment.affinity = 5 .job.environment.jit = RyuJit - .job.environment.runtime = Net47 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 53 [Entry33] @@ -1116,7 +1150,8 @@ .job.environment = .job.environment.affinity = 5 .job.environment.jit = RyuJit - .job.environment.runtime = Net47 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 56 [Entry34] @@ -1148,7 +1183,8 @@ .job.environment = .job.environment.affinity = 5 .job.environment.jit = RyuJit - .job.environment.runtime = Net47 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 57 [Entry35] @@ -1180,7 +1216,8 @@ .job.environment = .job.environment.affinity = 5 .job.environment.jit = RyuJit - .job.environment.runtime = Net47 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 58 [Entry36] @@ -1212,7 +1249,8 @@ .job.environment = .job.environment.affinity = 6 .job.environment.jit = RyuJit - .job.environment.runtime = Net471 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 61 [Entry37] @@ -1244,7 +1282,8 @@ .job.environment = .job.environment.affinity = 6 .job.environment.jit = RyuJit - .job.environment.runtime = Net471 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 62 [Entry38] @@ -1276,7 +1315,8 @@ .job.environment = .job.environment.affinity = 6 .job.environment.jit = RyuJit - .job.environment.runtime = Net471 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 63 [Entry39] @@ -1308,7 +1348,8 @@ .job.environment = .job.environment.affinity = 6 .job.environment.jit = RyuJit - .job.environment.runtime = Net471 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 66 [Entry40] @@ -1340,7 +1381,8 @@ .job.environment = .job.environment.affinity = 6 .job.environment.jit = RyuJit - .job.environment.runtime = Net471 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 67 [Entry41] @@ -1372,7 +1414,8 @@ .job.environment = .job.environment.affinity = 6 .job.environment.jit = RyuJit - .job.environment.runtime = Net471 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 68 [Entry42] @@ -1404,7 +1447,8 @@ .job.environment = .job.environment.affinity = 7 .job.environment.jit = RyuJit - .job.environment.runtime = Net472 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 71 [Entry43] @@ -1436,7 +1480,8 @@ .job.environment = .job.environment.affinity = 7 .job.environment.jit = RyuJit - .job.environment.runtime = Net472 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 72 [Entry44] @@ -1468,7 +1513,8 @@ .job.environment = .job.environment.affinity = 7 .job.environment.jit = RyuJit - .job.environment.runtime = Net472 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 73 [Entry45] @@ -1500,7 +1546,8 @@ .job.environment = .job.environment.affinity = 7 .job.environment.jit = RyuJit - .job.environment.runtime = Net472 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 76 [Entry46] @@ -1532,7 +1579,8 @@ .job.environment = .job.environment.affinity = 7 .job.environment.jit = RyuJit - .job.environment.runtime = Net472 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 77 [Entry47] @@ -1564,7 +1612,8 @@ .job.environment = .job.environment.affinity = 7 .job.environment.jit = RyuJit - .job.environment.runtime = Net472 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 78 [Entry48] @@ -1596,7 +1645,8 @@ .job.environment = .job.environment.affinity = 8 .job.environment.jit = RyuJit - .job.environment.runtime = Net48 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 81 [Entry49] @@ -1628,7 +1678,8 @@ .job.environment = .job.environment.affinity = 8 .job.environment.jit = RyuJit - .job.environment.runtime = Net48 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 82 [Entry50] @@ -1660,7 +1711,8 @@ .job.environment = .job.environment.affinity = 8 .job.environment.jit = RyuJit - .job.environment.runtime = Net48 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 83 [Entry51] @@ -1692,7 +1744,8 @@ .job.environment = .job.environment.affinity = 8 .job.environment.jit = RyuJit - .job.environment.runtime = Net48 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 86 [Entry52] @@ -1724,7 +1777,8 @@ .job.environment = .job.environment.affinity = 8 .job.environment.jit = RyuJit - .job.environment.runtime = Net48 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 87 [Entry53] @@ -1756,7 +1810,8 @@ .job.environment = .job.environment.affinity = 8 .job.environment.jit = RyuJit - .job.environment.runtime = Net48 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 88 [Entry54] @@ -1788,7 +1843,8 @@ .job.environment = .job.environment.affinity = 9 .job.environment.jit = RyuJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 91 [Entry55] @@ -1820,7 +1876,8 @@ .job.environment = .job.environment.affinity = 9 .job.environment.jit = RyuJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 92 [Entry56] @@ -1852,7 +1909,8 @@ .job.environment = .job.environment.affinity = 9 .job.environment.jit = RyuJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 93 [Entry57] @@ -1884,7 +1942,8 @@ .job.environment = .job.environment.affinity = 9 .job.environment.jit = RyuJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 96 [Entry58] @@ -1916,7 +1975,8 @@ .job.environment = .job.environment.affinity = 9 .job.environment.jit = RyuJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 97 [Entry59] @@ -1948,7 +2008,8 @@ .job.environment = .job.environment.affinity = 9 .job.environment.jit = RyuJit - .job.environment.runtime = Net481 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 98 [Entry60] @@ -1980,7 +2041,8 @@ .job.environment = .job.environment.affinity = 10 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp20 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 101 [Entry61] @@ -2012,7 +2074,8 @@ .job.environment = .job.environment.affinity = 10 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp20 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 102 [Entry62] @@ -2044,7 +2107,8 @@ .job.environment = .job.environment.affinity = 10 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp20 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 103 [Entry63] @@ -2076,7 +2140,8 @@ .job.environment = .job.environment.affinity = 10 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp20 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 106 [Entry64] @@ -2108,7 +2173,8 @@ .job.environment = .job.environment.affinity = 10 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp20 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 107 [Entry65] @@ -2140,7 +2206,8 @@ .job.environment = .job.environment.affinity = 10 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp20 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 108 [Entry66] @@ -2172,7 +2239,8 @@ .job.environment = .job.environment.affinity = 11 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp21 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 111 [Entry67] @@ -2204,7 +2272,8 @@ .job.environment = .job.environment.affinity = 11 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp21 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 112 [Entry68] @@ -2236,7 +2305,8 @@ .job.environment = .job.environment.affinity = 11 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp21 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 113 [Entry69] @@ -2268,7 +2338,8 @@ .job.environment = .job.environment.affinity = 11 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp21 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 116 [Entry70] @@ -2300,7 +2371,8 @@ .job.environment = .job.environment.affinity = 11 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp21 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 117 [Entry71] @@ -2332,7 +2404,8 @@ .job.environment = .job.environment.affinity = 11 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp21 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 118 [Entry72] @@ -2364,7 +2437,8 @@ .job.environment = .job.environment.affinity = 12 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp22 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 121 [Entry73] @@ -2396,7 +2470,8 @@ .job.environment = .job.environment.affinity = 12 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp22 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 122 [Entry74] @@ -2428,7 +2503,8 @@ .job.environment = .job.environment.affinity = 12 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp22 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 123 [Entry75] @@ -2460,7 +2536,8 @@ .job.environment = .job.environment.affinity = 12 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp22 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 126 [Entry76] @@ -2492,7 +2569,8 @@ .job.environment = .job.environment.affinity = 12 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp22 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 127 [Entry77] @@ -2524,7 +2602,8 @@ .job.environment = .job.environment.affinity = 12 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp22 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 128 [Entry78] @@ -2556,7 +2635,8 @@ .job.environment = .job.environment.affinity = 13 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp30 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 131 [Entry79] @@ -2588,7 +2668,8 @@ .job.environment = .job.environment.affinity = 13 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp30 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 132 [Entry80] @@ -2620,7 +2701,8 @@ .job.environment = .job.environment.affinity = 13 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp30 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 133 [Entry81] @@ -2652,7 +2734,8 @@ .job.environment = .job.environment.affinity = 13 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp30 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 136 [Entry82] @@ -2684,7 +2767,8 @@ .job.environment = .job.environment.affinity = 13 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp30 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 137 [Entry83] @@ -2716,7 +2800,8 @@ .job.environment = .job.environment.affinity = 13 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp30 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 138 [Entry84] @@ -2748,7 +2833,8 @@ .job.environment = .job.environment.affinity = 14 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp31 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 141 [Entry85] @@ -2780,7 +2866,8 @@ .job.environment = .job.environment.affinity = 14 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp31 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 142 [Entry86] @@ -2812,7 +2899,8 @@ .job.environment = .job.environment.affinity = 14 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp31 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 143 [Entry87] @@ -2844,7 +2932,8 @@ .job.environment = .job.environment.affinity = 14 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp31 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 146 [Entry88] @@ -2876,7 +2965,8 @@ .job.environment = .job.environment.affinity = 14 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp31 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 147 [Entry89] @@ -2908,7 +2998,8 @@ .job.environment = .job.environment.affinity = 14 .job.environment.jit = RyuJit - .job.environment.runtime = NetCoreApp31 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 148 [Entry90] @@ -2940,7 +3031,8 @@ .job.environment = .job.environment.affinity = 15 .job.environment.jit = RyuJit - .job.environment.runtime = Net50 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 151 [Entry91] @@ -2972,7 +3064,8 @@ .job.environment = .job.environment.affinity = 15 .job.environment.jit = RyuJit - .job.environment.runtime = Net50 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 152 [Entry92] @@ -3004,7 +3097,8 @@ .job.environment = .job.environment.affinity = 15 .job.environment.jit = RyuJit - .job.environment.runtime = Net50 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 153 [Entry93] @@ -3036,7 +3130,8 @@ .job.environment = .job.environment.affinity = 15 .job.environment.jit = RyuJit - .job.environment.runtime = Net50 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 156 [Entry94] @@ -3068,7 +3163,8 @@ .job.environment = .job.environment.affinity = 15 .job.environment.jit = RyuJit - .job.environment.runtime = Net50 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 157 [Entry95] @@ -3100,7 +3196,8 @@ .job.environment = .job.environment.affinity = 15 .job.environment.jit = RyuJit - .job.environment.runtime = Net50 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 158 [Entry96] @@ -3132,7 +3229,8 @@ .job.environment = .job.environment.affinity = 16 .job.environment.jit = RyuJit - .job.environment.runtime = Net60 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 161 [Entry97] @@ -3164,7 +3262,8 @@ .job.environment = .job.environment.affinity = 16 .job.environment.jit = RyuJit - .job.environment.runtime = Net60 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 162 [Entry98] @@ -3196,7 +3295,8 @@ .job.environment = .job.environment.affinity = 16 .job.environment.jit = RyuJit - .job.environment.runtime = Net60 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 163 [Entry99] @@ -3228,7 +3328,8 @@ .job.environment = .job.environment.affinity = 16 .job.environment.jit = RyuJit - .job.environment.runtime = Net60 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 166 [Entry100] @@ -3260,7 +3361,8 @@ .job.environment = .job.environment.affinity = 16 .job.environment.jit = RyuJit - .job.environment.runtime = Net60 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 167 [Entry101] @@ -3292,7 +3394,8 @@ .job.environment = .job.environment.affinity = 16 .job.environment.jit = RyuJit - .job.environment.runtime = Net60 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 168 [Entry102] @@ -3324,7 +3427,8 @@ .job.environment = .job.environment.affinity = 17 .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 171 [Entry103] @@ -3356,7 +3460,8 @@ .job.environment = .job.environment.affinity = 17 .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 172 [Entry104] @@ -3388,7 +3493,8 @@ .job.environment = .job.environment.affinity = 17 .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 173 [Entry105] @@ -3420,7 +3526,8 @@ .job.environment = .job.environment.affinity = 17 .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 176 [Entry106] @@ -3452,7 +3559,8 @@ .job.environment = .job.environment.affinity = 17 .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 177 [Entry107] @@ -3484,7 +3592,8 @@ .job.environment = .job.environment.affinity = 17 .job.environment.jit = RyuJit - .job.environment.runtime = Net70 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 178 [Entry108] @@ -3516,7 +3625,8 @@ .job.environment = .job.environment.affinity = 18 .job.environment.jit = RyuJit - .job.environment.runtime = Net80 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 181 [Entry109] @@ -3548,7 +3658,8 @@ .job.environment = .job.environment.affinity = 18 .job.environment.jit = RyuJit - .job.environment.runtime = Net80 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 182 [Entry110] @@ -3580,7 +3691,8 @@ .job.environment = .job.environment.affinity = 18 .job.environment.jit = RyuJit - .job.environment.runtime = Net80 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 183 [Entry111] @@ -3612,7 +3724,8 @@ .job.environment = .job.environment.affinity = 18 .job.environment.jit = RyuJit - .job.environment.runtime = Net80 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 186 [Entry112] @@ -3644,7 +3757,8 @@ .job.environment = .job.environment.affinity = 18 .job.environment.jit = RyuJit - .job.environment.runtime = Net80 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 187 [Entry113] @@ -3676,7 +3790,8 @@ .job.environment = .job.environment.affinity = 18 .job.environment.jit = RyuJit - .job.environment.runtime = Net80 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 188 [Entry114] @@ -3708,7 +3823,8 @@ .job.environment = .job.environment.affinity = 19 .job.environment.jit = RyuJit - .job.environment.runtime = Net90 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 191 [Entry115] @@ -3740,7 +3856,8 @@ .job.environment = .job.environment.affinity = 19 .job.environment.jit = RyuJit - .job.environment.runtime = Net90 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 192 [Entry116] @@ -3772,7 +3889,8 @@ .job.environment = .job.environment.affinity = 19 .job.environment.jit = RyuJit - .job.environment.runtime = Net90 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 193 [Entry117] @@ -3804,7 +3922,8 @@ .job.environment = .job.environment.affinity = 19 .job.environment.jit = RyuJit - .job.environment.runtime = Net90 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 196 [Entry118] @@ -3836,7 +3955,8 @@ .job.environment = .job.environment.affinity = 19 .job.environment.jit = RyuJit - .job.environment.runtime = Net90 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 197 [Entry119] @@ -3868,6 +3988,7 @@ .job.environment = .job.environment.affinity = 19 .job.environment.jit = RyuJit - .job.environment.runtime = Net90 + .job.infrastructure = + .job.infrastructure.runtime = net7.0 .unit = ns .value = 198 \ No newline at end of file diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt index dfbf9bdcaa..e445c5c934 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default02.verified.txt @@ -5,10 +5,10 @@ Type = Bench | Method | Runtime | Center | Spread | |:-------|:--------|-------:|-------:| -| Foo | Net481 | 11.0ns | 0.81ns | -| Bar | Net481 | 21.0ns | 0.81ns | -| Foo | Net70 | 31.0ns | 0.81ns | -| Bar | Net70 | 41.0ns | 0.81ns | +| Foo | net481 | 11.0ns | 0.81ns | +| Bar | net481 | 21.0ns | 0.81ns | +| Foo | net7.0 | 31.0ns | 0.81ns | +| Bar | net7.0 | 41.0ns | 0.81ns | { "engine": { @@ -38,8 +38,10 @@ Type = Bench "nested": [ { "job": { - "environment": { + "infrastructure": { "runtime": "net481" + }, + "environment": { } }, "nested": [ @@ -93,8 +95,10 @@ Type = Bench }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net70" } }, "nested": [ diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt index 6d4df1ab14..3f9c0552ee 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default03.verified.txt @@ -1,7 +1,7 @@ BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Type = Bench, Runtime = Net70, Jit = RyuJit +Type = Bench, Runtime = net7.0, Jit = RyuJit | Method | Center | Spread | |:-------|-------:|-------:| @@ -36,8 +36,10 @@ Type = Bench, Runtime = Net70, Jit = RyuJit "nested": [ { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net70", "jit": "ryuJit" } }, diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt index a021bf7ce1..e36b933ee9 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default04.verified.txt @@ -3,15 +3,15 @@ MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 phys Type = Bench -@Net481LegacyJit: Runtime = Net481, Jit = LegacyJit -@Net70RyuJit: Runtime = Net70, Jit = RyuJit +@Mussel: Runtime = net481, Jit = LegacyJit +@Runtimenet70RyuJit: Runtime = net7.0, Jit = RyuJit -| Method | Job | Center | Spread | -|:-------|:----------------|-------:|-------:| -| Foo | Net481LegacyJit | 11.0ns | 0.81ns | -| Bar | Net481LegacyJit | 21.0ns | 0.81ns | -| Foo | Net70RyuJit | 31.0ns | 0.81ns | -| Bar | Net70RyuJit | 41.0ns | 0.81ns | +| Method | Job | Center | Spread | +|:-------|:-------------------|-------:|-------:| +| Foo | Mussel | 11.0ns | 0.81ns | +| Bar | Mussel | 21.0ns | 0.81ns | +| Foo | Runtimenet70RyuJit | 31.0ns | 0.81ns | +| Bar | Runtimenet70RyuJit | 41.0ns | 0.81ns | { "engine": { @@ -41,8 +41,10 @@ Type = Bench "nested": [ { "job": { + "infrastructure": { + "runtime": "net481" + }, "environment": { - "runtime": "net481", "jit": "legacyJit" } }, @@ -97,8 +99,10 @@ Type = Bench }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net70", "jit": "ryuJit" } }, diff --git a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt index d5d49c5ab8..56bbca3f01 100644 --- a/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt +++ b/tests/BenchmarkDotNet.Tests/Perfonar/VerifiedFiles/Perfonar.PerfonarTableTest_key=default05.verified.txt @@ -1,71 +1,50 @@ BenchmarkDotNet v0.1729.0-mock, Microsoft Windows NT 10.0.x.mock MockIntel Core i7-6700HQ CPU 2.60GHz (Max: 3.10GHz), 1 CPU, 8 logical and 4 physical cores -Type = Bench, Jit = RyuJit +Type = Bench, Runtime = net7.0, Jit = RyuJit -@HostProcessAffinity0: Runtime = HostProcess, Affinity = 0 -@Squid: Runtime = NotRecognized, Affinity = 1 -@MonoAffinity2: Runtime = Mono, Affinity = 2 -@Net461Affinity3: Runtime = Net461, Affinity = 3 -@Net462Affinity4: Runtime = Net462, Affinity = 4 -@Net47Affinity5: Runtime = Net47, Affinity = 5 -@Net471Affinity6: Runtime = Net471, Affinity = 6 -@Net472Affinity7: Runtime = Net472, Affinity = 7 -@Net48Affinity8: Runtime = Net48, Affinity = 8 -@Net481Affinity9: Runtime = Net481, Affinity = 9 -@Liger: Runtime = NetCoreApp20, Affinity = 10 -@Rat: Runtime = NetCoreApp21, Affinity = 11 -@Skate: Runtime = NetCoreApp22, Affinity = 12 -@Perch: Runtime = NetCoreApp30, Affinity = 13 -@Roach: Runtime = NetCoreApp31, Affinity = 14 -@Net50Affinity15: Runtime = Net50, Affinity = 15 -@Net60Affinity16: Runtime = Net60, Affinity = 16 -@Net70Affinity17: Runtime = Net70, Affinity = 17 -@Net80Affinity18: Runtime = Net80, Affinity = 18 -@Net90Affinity19: Runtime = Net90, Affinity = 19 - -| Method | Job | Center | Spread | -|:-------|:---------------------|---------:|-------:| -| Foo | HostProcessAffinity0 | 2.00ns | 0.81ns | -| Bar | HostProcessAffinity0 | 7.00ns | 0.81ns | -| Foo | Squid | 12.00ns | 0.81ns | -| Bar | Squid | 17.00ns | 0.81ns | -| Foo | MonoAffinity2 | 22.00ns | 0.81ns | -| Bar | MonoAffinity2 | 27.00ns | 0.81ns | -| Foo | Net461Affinity3 | 32.00ns | 0.81ns | -| Bar | Net461Affinity3 | 37.00ns | 0.81ns | -| Foo | Net462Affinity4 | 42.00ns | 0.81ns | -| Bar | Net462Affinity4 | 47.00ns | 0.81ns | -| Foo | Net47Affinity5 | 52.00ns | 0.81ns | -| Bar | Net47Affinity5 | 57.00ns | 0.81ns | -| Foo | Net471Affinity6 | 62.00ns | 0.81ns | -| Bar | Net471Affinity6 | 67.00ns | 0.81ns | -| Foo | Net472Affinity7 | 72.00ns | 0.81ns | -| Bar | Net472Affinity7 | 77.00ns | 0.81ns | -| Foo | Net48Affinity8 | 82.00ns | 0.81ns | -| Bar | Net48Affinity8 | 87.00ns | 0.81ns | -| Foo | Net481Affinity9 | 92.00ns | 0.81ns | -| Bar | Net481Affinity9 | 97.00ns | 0.81ns | -| Foo | Liger | 102.00ns | 0.81ns | -| Bar | Liger | 107.00ns | 0.81ns | -| Foo | Rat | 112.00ns | 0.81ns | -| Bar | Rat | 117.00ns | 0.81ns | -| Foo | Skate | 122.00ns | 0.81ns | -| Bar | Skate | 127.00ns | 0.81ns | -| Foo | Perch | 132.00ns | 0.81ns | -| Bar | Perch | 137.00ns | 0.81ns | -| Foo | Roach | 142.00ns | 0.81ns | -| Bar | Roach | 147.00ns | 0.81ns | -| Foo | Net50Affinity15 | 152.00ns | 0.81ns | -| Bar | Net50Affinity15 | 157.00ns | 0.81ns | -| Foo | Net60Affinity16 | 162.00ns | 0.81ns | -| Bar | Net60Affinity16 | 167.00ns | 0.81ns | -| Foo | Net70Affinity17 | 172.00ns | 0.81ns | -| Bar | Net70Affinity17 | 177.00ns | 0.81ns | -| Foo | Net80Affinity18 | 182.00ns | 0.81ns | -| Bar | Net80Affinity18 | 187.00ns | 0.81ns | -| Foo | Net90Affinity19 | 192.00ns | 0.81ns | -| Bar | Net90Affinity19 | 197.00ns | 0.81ns | +| Method | Affinity | Center | Spread | +|:-------|:---------|---------:|-------:| +| Foo | 0 | 2.00ns | 0.81ns | +| Bar | 0 | 7.00ns | 0.81ns | +| Foo | 1 | 12.00ns | 0.81ns | +| Bar | 1 | 17.00ns | 0.81ns | +| Foo | 2 | 22.00ns | 0.81ns | +| Bar | 2 | 27.00ns | 0.81ns | +| Foo | 3 | 32.00ns | 0.81ns | +| Bar | 3 | 37.00ns | 0.81ns | +| Foo | 4 | 42.00ns | 0.81ns | +| Bar | 4 | 47.00ns | 0.81ns | +| Foo | 5 | 52.00ns | 0.81ns | +| Bar | 5 | 57.00ns | 0.81ns | +| Foo | 6 | 62.00ns | 0.81ns | +| Bar | 6 | 67.00ns | 0.81ns | +| Foo | 7 | 72.00ns | 0.81ns | +| Bar | 7 | 77.00ns | 0.81ns | +| Foo | 8 | 82.00ns | 0.81ns | +| Bar | 8 | 87.00ns | 0.81ns | +| Foo | 9 | 92.00ns | 0.81ns | +| Bar | 9 | 97.00ns | 0.81ns | +| Foo | 10 | 102.00ns | 0.81ns | +| Bar | 10 | 107.00ns | 0.81ns | +| Foo | 11 | 112.00ns | 0.81ns | +| Bar | 11 | 117.00ns | 0.81ns | +| Foo | 12 | 122.00ns | 0.81ns | +| Bar | 12 | 127.00ns | 0.81ns | +| Foo | 13 | 132.00ns | 0.81ns | +| Bar | 13 | 137.00ns | 0.81ns | +| Foo | 14 | 142.00ns | 0.81ns | +| Bar | 14 | 147.00ns | 0.81ns | +| Foo | 15 | 152.00ns | 0.81ns | +| Bar | 15 | 157.00ns | 0.81ns | +| Foo | 16 | 162.00ns | 0.81ns | +| Bar | 16 | 167.00ns | 0.81ns | +| Foo | 17 | 172.00ns | 0.81ns | +| Bar | 17 | 177.00ns | 0.81ns | +| Foo | 18 | 182.00ns | 0.81ns | +| Bar | 18 | 187.00ns | 0.81ns | +| Foo | 19 | 192.00ns | 0.81ns | +| Bar | 19 | 197.00ns | 0.81ns | { "engine": { @@ -95,8 +74,10 @@ Type = Bench, Jit = RyuJit "nested": [ { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "hostProcess", "jit": "ryuJit", "affinity": 0 } @@ -152,8 +133,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "notRecognized", "jit": "ryuJit", "affinity": 1 } @@ -209,8 +192,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "mono", "jit": "ryuJit", "affinity": 2 } @@ -266,8 +251,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net461", "jit": "ryuJit", "affinity": 3 } @@ -323,8 +310,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net462", "jit": "ryuJit", "affinity": 4 } @@ -380,8 +369,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net47", "jit": "ryuJit", "affinity": 5 } @@ -437,8 +428,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net471", "jit": "ryuJit", "affinity": 6 } @@ -494,8 +487,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net472", "jit": "ryuJit", "affinity": 7 } @@ -551,8 +546,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net48", "jit": "ryuJit", "affinity": 8 } @@ -608,8 +605,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net481", "jit": "ryuJit", "affinity": 9 } @@ -665,8 +664,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "netCoreApp20", "jit": "ryuJit", "affinity": 10 } @@ -722,8 +723,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "netCoreApp21", "jit": "ryuJit", "affinity": 11 } @@ -779,8 +782,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "netCoreApp22", "jit": "ryuJit", "affinity": 12 } @@ -836,8 +841,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "netCoreApp30", "jit": "ryuJit", "affinity": 13 } @@ -893,8 +900,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "netCoreApp31", "jit": "ryuJit", "affinity": 14 } @@ -950,8 +959,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net50", "jit": "ryuJit", "affinity": 15 } @@ -1007,8 +1018,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net60", "jit": "ryuJit", "affinity": 16 } @@ -1064,8 +1077,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net70", "jit": "ryuJit", "affinity": 17 } @@ -1121,8 +1136,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net80", "jit": "ryuJit", "affinity": 18 } @@ -1178,8 +1195,10 @@ Type = Bench, Jit = RyuJit }, { "job": { + "infrastructure": { + "runtime": "net7.0" + }, "environment": { - "runtime": "net90", "jit": "ryuJit", "affinity": 19 } diff --git a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs index 962c41d925..079d4260ac 100644 --- a/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs +++ b/tests/BenchmarkDotNet.Tests/Reports/SummaryTests.cs @@ -43,7 +43,7 @@ private static BenchmarkReport[] CreateReports(IConfig config) private static BenchmarkReport CreateReport(BenchmarkCase benchmark) { - return benchmark.Job.Environment.Runtime is ClrRuntime + return benchmark.GetRuntime() is ClrRuntime ? CreateFailureReport(benchmark) : CreateSuccessReport(benchmark); } diff --git a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs index ccd53faa5d..6cdb1f11c6 100644 --- a/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/BenchmarkConverterTests.cs @@ -2,6 +2,7 @@ using BenchmarkDotNet.Configs; using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Toolchains; using BenchmarkDotNet.Running; using Perfolizer.Mathematics.OutlierDetection; @@ -136,8 +137,8 @@ public void JobMutatorsApplySettingsToAllNonMutatorJobs() Assert.Equal(2, info.BenchmarksCases.Length); Assert.All(info.BenchmarksCases, benchmark => Assert.Equal(int.MaxValue, benchmark.Job.Run.MaxIterationCount)); - Assert.Single(info.BenchmarksCases, benchmark => benchmark.Job.Environment.Runtime is ClrRuntime); - Assert.Single(info.BenchmarksCases, benchmark => benchmark.Job.Environment.Runtime is CoreRuntime); + Assert.Single(info.BenchmarksCases, benchmark => benchmark.GetRuntime() is ClrRuntime); + Assert.Single(info.BenchmarksCases, benchmark => benchmark.GetRuntime() is CoreRuntime); Assert.All(info.BenchmarksCases, benchmark => Assert.False(benchmark.Job.Meta.IsMutator)); // the job does not became a mutator itself, this config should not be copied } @@ -166,7 +167,7 @@ public void OrderOfAppliedAttributesDoesNotAffectMutators() var benchmark = info.BenchmarksCases.Single(); Assert.Equal(int.MaxValue, benchmark.Job.Run.MaxIterationCount); - Assert.True(benchmark.Job.Environment.Runtime is CoreRuntime); + Assert.True(benchmark.GetRuntime() is CoreRuntime); Assert.False(benchmark.Job.Meta.IsMutator); } diff --git a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs index b926b0bd05..f9ca0bcf98 100644 --- a/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs +++ b/tests/BenchmarkDotNet.Tests/Running/JobRuntimePropertiesComparerTests.cs @@ -4,7 +4,7 @@ using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using BenchmarkDotNet.Tests.XUnit; -using BenchmarkDotNet.Toolchains.CsProj; +using BenchmarkDotNet.Toolchains.NetCoreApp; namespace BenchmarkDotNet.Tests.Running { @@ -69,45 +69,16 @@ [Benchmark] public void M1() { } [Benchmark] public void M2() { } } - [FactEnvSpecific("Full Framework is supported only on Windows", EnvRequirement.WindowsOnly)] - public void CustomClrBuildJobsAreGroupedByVersion() - { - const string version = "abcd"; - - var config = ManualConfig.Create(DefaultConfig.Instance) - .AddJob(Job.Default.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: version))) - .AddJob(Job.Default.WithRuntime(ClrRuntime.CreateForLocalFullNetFrameworkBuild(version: "it's a different version"))) - .AddJob(Job.Default.WithRuntime(ClrRuntime.GetCurrentVersion())); - - var benchmarks1 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain1), config); - var benchmarks2 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain2), config); - - var grouped = benchmarks1.BenchmarksCases.Union(benchmarks2.BenchmarksCases) - .GroupBy(benchmark => benchmark, new BenchmarkPartitioner.BenchmarkRuntimePropertiesComparer()) - .ToArray(); - - Assert.Equal(3, grouped.Length); // Job.Clr + Job.Clr(version) + Job.Clr(different) - - foreach (var grouping in grouped) - Assert.Equal(3 * 2, grouping.Count()); // (M1 + M2 + M3) * (Plain1 + Plain2) - } - [Fact] public void CustomTargetPlatformJobsAreGroupedByTargetFrameworkMoniker() { var net5Config = ManualConfig.Create(DefaultConfig.Instance) .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp50)); var net5WindowsConfig1 = ManualConfig.Create(DefaultConfig.Instance) - .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.From(new Toolchains.DotNetCli.NetCoreAppSettings( - targetFrameworkMoniker: "net5.0-windows", - runtimeFrameworkVersion: null!, - name: ".NET 5.0")))); + .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core50, new NetCoreAppSettings { TargetFrameworkMoniker = "net5.0-windows" }))); // a different INSTANCE of CsProjCoreToolchain that also targets "net5.0-windows" var net5WindowsConfig2 = ManualConfig.Create(DefaultConfig.Instance) - .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.From(new Toolchains.DotNetCli.NetCoreAppSettings( - targetFrameworkMoniker: "net5.0-windows", - runtimeFrameworkVersion: null!, - name: ".NET 5.0")))); + .AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.From(CoreRuntime.Core50, new NetCoreAppSettings { TargetFrameworkMoniker = "net5.0-windows" }))); var benchmarksNet5 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain1), net5Config); var benchmarksNet5Windows1 = BenchmarkConverter.TypeToBenchmarks(typeof(Plain2), net5WindowsConfig1); diff --git a/tests/BenchmarkDotNet.Tests/RuntimeEqualityTests.cs b/tests/BenchmarkDotNet.Tests/RuntimeEqualityTests.cs new file mode 100644 index 0000000000..5315e839b5 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/RuntimeEqualityTests.cs @@ -0,0 +1,41 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Tests; + +public class RuntimeEqualityTests +{ + // A custom runtime with equality state (Extra) beyond Name/Version, overriding the polymorphic Equals(object). + private sealed class CustomRuntime(string name, Version version, string extra) : Runtime + { + public override string Name => name; + public override Version? Version => version; + private string Extra { get; } = extra; + + public override IToolchain GetDefaultToolchain(BenchmarkCase benchmarkCase) => throw new NotImplementedException(); + public override bool Equals(object? obj) => obj is CustomRuntime other && base.Equals(other) && Extra == other.Extra; + public override int GetHashCode() => HashCode.Combine(base.GetHashCode(), Extra); + } + + [Fact] + public void CustomRuntimeEqualityIsHonoredThroughTheIEquatableRuntimePath() + { + Runtime a = new CustomRuntime("Custom", new Version(1, 0), "a"); + Runtime differsByExtra = new CustomRuntime("Custom", new Version(1, 0), "b"); + Runtime sameAsA = new CustomRuntime("Custom", new Version(1, 0), "a"); + + // Directly via the concrete Equals(object). + Assert.False(a.Equals((object)differsByExtra)); + Assert.True(a.Equals((object)sameAsA)); + + // Via IEquatable (Dictionary/HashSet/EqualityComparer use this) — the path that previously ignored Extra. + var comparer = EqualityComparer.Default; + Assert.False(comparer.Equals(a, differsByExtra)); + Assert.True(comparer.Equals(a, sameAsA)); + + var dictionary = new Dictionary { [a] = 1 }; + Assert.False(dictionary.ContainsKey(differsByExtra)); + Assert.True(dictionary.ContainsKey(sameAsA)); + } +} diff --git a/tests/BenchmarkDotNet.Tests/RuntimeParseTests.cs b/tests/BenchmarkDotNet.Tests/RuntimeParseTests.cs new file mode 100644 index 0000000000..d66c9b9e8e --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/RuntimeParseTests.cs @@ -0,0 +1,262 @@ +using BenchmarkDotNet.Environments; +using BenchmarkDotNet.Toolchains; + +namespace BenchmarkDotNet.Tests; + +public class RuntimeParseTests +{ + [Theory] + // dotted / CLI spellings + [InlineData("net8.0", typeof(CoreRuntime), 8, 0)] + [InlineData("net10.0", typeof(CoreRuntime), 10, 0)] + [InlineData("netcoreapp3.1", typeof(CoreRuntime), 3, 1)] + [InlineData("net472", typeof(ClrRuntime), 4, 7)] + [InlineData("net48", typeof(ClrRuntime), 4, 8)] + [InlineData("nativeaot8.0", typeof(NativeAotRuntime), 8, 0)] + [InlineData("mono8.0", typeof(MonoCoreRuntime), 8, 0)] + [InlineData("monowasm8.0", typeof(MonoWasmRuntime), 8, 0)] + [InlineData("monowasmAot8.0", typeof(MonoWasmAotRuntime), 8, 0)] + [InlineData("corewasm11.0", typeof(CoreWasmRuntime), 11, 0)] + [InlineData("r2r8.0", typeof(R2RRuntime), 8, 0)] + // compact / enum-name spellings + [InlineData("Net80", typeof(CoreRuntime), 8, 0)] + [InlineData("Net10_0", typeof(CoreRuntime), 10, 0)] + [InlineData("NetCoreApp31", typeof(CoreRuntime), 3, 1)] + [InlineData("NativeAot70", typeof(NativeAotRuntime), 7, 0)] + [InlineData("Mono60", typeof(MonoCoreRuntime), 6, 0)] + [InlineData("MonoWasm80", typeof(MonoWasmRuntime), 8, 0)] + [InlineData("MonoWasmAot80", typeof(MonoWasmAotRuntime), 8, 0)] + [InlineData("CoreWasm11_0", typeof(CoreWasmRuntime), 11, 0)] + [InlineData("R2R80", typeof(R2RRuntime), 8, 0)] + public void ParsesKnownMonikers(string moniker, Type expectedType, int major, int minor) + { + Assert.True(Runtime.TryParse(moniker, out var runtime)); + Assert.IsType(expectedType, runtime); + Assert.Equal(major, runtime!.Version!.Major); + Assert.Equal(minor, runtime.Version.Minor); + } + + [Theory] + [InlineData("")] + [InlineData(" ")] + [InlineData("hostprocess")] + [InlineData("notrecognized")] + [InlineData("garbage")] + [InlineData("net")] + // dropped legacy wasm spellings — no longer recognized (only monowasm/monowasmAot/corewasm are) + [InlineData("wasmnet8.0")] + [InlineData("WasmNet80")] + [InlineData("WasmCoreNet11_0")] + // legacy Mono AOT is versionless; the new Mono AOT has no public toolchain + [InlineData("monoaot8.0")] + public void RejectsNonRuntimeMonikers(string moniker) + { + Assert.False(Runtime.TryParse(moniker, out var runtime)); + Assert.Null(runtime); + } + + [Theory] + [InlineData("net5.0-windows", "net5.0-windows")] + [InlineData("net8.0-ios", "net8.0-ios")] + [InlineData("net10.0-windows10.0.19041.0", "net10.0-windows10.0.19041.0")] + // The platform is never normalized, so the casing survives into the moniker. + [InlineData("net10.0-WINDOWS", "net10.0-WINDOWS")] + public void PreservesPlatformSpecificMonikers(string moniker, string expectedTfm) + { + Assert.True(Runtime.TryParse(moniker, out var runtime)); + var core = Assert.IsType(runtime); + Assert.True(core.IsPlatformSpecific); + Assert.Equal(expectedTfm, core.GetTfm()); + } + + [Theory] + // The platform is substituted verbatim into the generated project's : '<' would produce + // malformed XML and ';' a second target framework, so neither may reach it. + [InlineData("net8.0-")] + [InlineData("net8.0-a;net9.0")] + [InlineData("net8.0-my platform")] + [InlineData("net8.0-win/dows")] + [InlineData("net8.0-\"windows\"")] + [InlineData("net8.0-")] + [InlineData("net8.0-8.0")] // a version with no platform name + // Only .NET (Core) carries a platform; the others used to drop the suffix and silently benchmark something else. + [InlineData("nativeaot8.0-windows")] + [InlineData("monowasm8.0-browser")] + [InlineData("net472-windows")] + // Only .NET 5.0 and later have platform-specific monikers. These have to be reported as unparseable, not throw: + // TryParse is what the command line calls to decide whether to print its "invalid runtime" message. + [InlineData("net3.1-windows")] + [InlineData("net2.1-ios")] + [InlineData("net3.0-windows10.0.19041.0")] + [InlineData("netcoreapp3.1-windows")] + public void RejectsMalformedOrUnsupportedPlatformSuffixes(string moniker) + { + Assert.False(Runtime.TryParse(moniker, out var runtime)); + Assert.Null(runtime); + } + + [Theory] + // Every runtime whose version identifies a .NET release, not just CoreRuntime: they all take a version straight + // from Environment.Version somewhere, and their From() switches only normalize the majors they know about. + [InlineData("nativeaot")] + [InlineData("r2r")] + [InlineData("mono")] + [InlineData("monowasm")] + [InlineData("monowasmaot")] + [InlineData("corewasm")] + public void EveryDotNetRuntimeTrimsAnythingPastMajorMinor(string prefix) + { + // A major outside every From() switch, so the uncached fallback is what runs. + var trimmed = Runtime.Parse($"{prefix}99.0"); + var withBuild = prefix switch + { + "nativeaot" => NativeAotRuntime.From(new Version(99, 0, 30)), + "r2r" => R2RRuntime.From(new Version(99, 0, 30)), + "mono" => MonoCoreRuntime.From(new Version(99, 0, 30)), + "monowasm" => MonoWasmRuntime.From(new Version(99, 0, 30)), + "monowasmaot" => MonoWasmAotRuntime.From(new Version(99, 0, 30)), + _ => (Runtime) CoreWasmRuntime.From(new Version(99, 0, 30)), + }; + + Assert.Equal(new Version(99, 0), withBuild.Version); + Assert.Equal(trimmed, withBuild); + Assert.Equal(trimmed.GetHashCode(), withBuild.GetHashCode()); + Assert.Equal(trimmed.ToString(), withBuild.ToString()); + } + + [Theory] + // .NET Framework keeps its build: ClrRuntime builds its moniker from it, so 4.8.1 has to stay distinguishable + // from 4.8. Only the revision is servicing detail there. + [InlineData("net481", 4, 8, 1)] + [InlineData("net472", 4, 7, 2)] + [InlineData("net48", 4, 8, -1)] + public void ClrRuntimeKeepsItsBuildVersion(string moniker, int major, int minor, int build) + { + var runtime = Assert.IsType(Runtime.Parse(moniker)); + Assert.Equal(major, runtime.Version.Major); + Assert.Equal(minor, runtime.Version.Minor); + Assert.Equal(build, runtime.Version.Build); + Assert.Equal(-1, runtime.Version.Revision); + } + + [Theory] + // A revision never identifies a .NET Framework release, and GetTfm() ignores it, so keeping it would make two + // runtimes with the same moniker compare unequal. 4.5 is below the supported floor, so it reaches the uncached + // fallback in FromVersion rather than a cached instance. + [InlineData(4, 5, 0, 0)] + [InlineData(4, 5, 1, 7)] + public void ClrRuntimeTrimsTheRevision(int major, int minor, int build, int revision) + { + var withRevision = ClrRuntime.FromVersion(new Version(major, minor, build, revision)); + var without = ClrRuntime.FromVersion(build > 0 ? new Version(major, minor, build) : new Version(major, minor)); + + Assert.Equal(-1, withRevision.Version.Revision); + Assert.Equal(without, withRevision); + Assert.Equal(without.GetHashCode(), withRevision.GetHashCode()); + Assert.Equal(without.ToString(), withRevision.ToString()); + } + + [Theory] + [InlineData("")] + [InlineData("a;net9.0")] + [InlineData("my platform")] + [InlineData("win/dows")] + [InlineData("8.0")] + // Every dot has to separate two digits, or the moniker built from it is one MSBuild rejects. + [InlineData("windows.")] + [InlineData("windows..0")] + [InlineData("windows.0")] + [InlineData("windows10.")] + public void CoreRuntimeFromRejectsMalformedPlatform(string platform) + { + // The parser is not the only way in: CoreRuntime.From is public, and the platform it is given ends up in the + // generated project's TargetFrameworks just the same. + var exception = Assert.Throws(() => CoreRuntime.From(new Version(8, 0), platform)); + Assert.Equal("platform", exception.ParamName); + } + + [Theory] + // Version(8,0), Version(8,0,0) and Version(8,0,0,0) all describe .NET 8.0, and Environment.Version supplies the + // servicing build (8.0.30) on the auto-detection path, so all four have to collapse to one runtime - otherwise + // the same runtime compares unequal to itself and jobs are duplicated. + [InlineData(0, -1)] + [InlineData(0, 0)] + [InlineData(30, -1)] + [InlineData(30, 5)] + public void TrimsAnythingPastMajorMinor(int build, int revision) + { + var version = revision >= 0 ? new Version(8, 0, build, revision) + : build >= 0 ? new Version(8, 0, build) + : new Version(8, 0); + + var runtime = CoreRuntime.From(version); + Assert.Equal(new Version(8, 0), runtime.Version); + Assert.Equal(".NET 8.0", runtime.ToString()); + Assert.Equal(CoreRuntime.Core80, runtime); + Assert.Equal(CoreRuntime.Core80.GetHashCode(), runtime.GetHashCode()); + + // Same again for the platform-specific path, which does not go through the cached instances. + var platformSpecific = CoreRuntime.From(version, "windows"); + Assert.Equal(new Version(8, 0), platformSpecific.Version); + Assert.Equal(CoreRuntime.From(new Version(8, 0), "windows"), platformSpecific); + } + + [Theory] + [InlineData("windows")] + [InlineData("WINDOWS")] + [InlineData("windows10.0.19041.0")] + [InlineData("windows10")] + [InlineData("ios")] + public void CoreRuntimeFromAcceptsWellFormedPlatform(string platform) + { + var runtime = CoreRuntime.From(new Version(8, 0), platform); + Assert.True(runtime.IsPlatformSpecific); + Assert.Equal(platform, runtime.Platform); + } + + [Theory] + [InlineData(2, 1)] + [InlineData(3, 1)] + public void CoreRuntimeFromRejectsPlatformOnVersionsWithoutPlatformMonikers(int major, int minor) + { + // netcoreappX.Y has no platform-specific form, so GetTfm() would drop the platform and silently build for + // something other than what was asked for. + var exception = Assert.Throws(() => CoreRuntime.From(new Version(major, minor), "windows")); + Assert.Equal("platform", exception.ParamName); + } + + [Fact] + public void ParsesToTheSameInstanceIdentityAsKnownStatics() + { + Assert.Equal(ClrRuntime.Net472, Runtime.Parse("net472")); + // versionless singletons + Assert.Equal(MonoRuntime.Default, Runtime.Parse("mono")); + Assert.Equal(MonoAotRuntime.Default, Runtime.Parse("monoaot")); + Assert.Equal(MonoWasmRuntime.Net80, Runtime.Parse("monowasm8.0")); + Assert.Equal(MonoWasmAotRuntime.Net80, Runtime.Parse("monowasmAot8.0")); + // CoreWasm has no cached static (still experimental), so compare against an allocated instance. + Assert.Equal(CoreWasmRuntime.From(new Version(11, 0)), Runtime.Parse("CoreWasm11_0")); + Assert.Equal(NativeAotRuntime.Net80, Runtime.Parse("nativeaot8.0")); + } + + [Theory] + // Every example the "invalid runtime" error advertises, so the message cannot drift from the parser. If a form is + // added there, add it here; if one stops parsing, this fails rather than the CLI recommending something invalid. + [InlineData("net472")] + [InlineData("net8.0")] + [InlineData("net8.0-windows")] + [InlineData("netcoreapp3.1")] + [InlineData("nativeaot8.0")] + [InlineData("r2r8.0")] + [InlineData("mono")] + [InlineData("mono8.0")] + [InlineData("monoaot")] + [InlineData("monowasm8.0")] + [InlineData("monowasmaot8.0")] + [InlineData("corewasm11.0")] + public void EveryMonikerFormTheErrorMessageAdvertisesParses(string moniker) + { + Assert.True(Runtime.TryParse(moniker, out var runtime), $"'{moniker}' is advertised by the invalid-runtime error but does not parse."); + Assert.NotNull(runtime); + } +} diff --git a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs index 31ab7e9e0b..bc0ee9e031 100644 --- a/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs +++ b/tests/BenchmarkDotNet.Tests/RuntimeVersionDetectionTests.cs @@ -1,34 +1,35 @@ using BenchmarkDotNet.Detectors; using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Portability; +using BenchmarkDotNet.Toolchains; namespace BenchmarkDotNet.Tests { public class RuntimeVersionDetectionTests { [Theory] - [InlineData(".NETCoreApp,Version=v2.0", RuntimeMoniker.NetCoreApp20, "netcoreapp2.0")] - [InlineData(".NETCoreApp,Version=v2.1", RuntimeMoniker.NetCoreApp21, "netcoreapp2.1")] - [InlineData(".NETCoreApp,Version=v2.2", RuntimeMoniker.NetCoreApp22, "netcoreapp2.2")] - [InlineData(".NETCoreApp,Version=v3.0", RuntimeMoniker.NetCoreApp30, "netcoreapp3.0")] - [InlineData(".NETCoreApp,Version=v3.1", RuntimeMoniker.NetCoreApp31, "netcoreapp3.1")] - [InlineData(".NETCoreApp,Version=v5.0", RuntimeMoniker.Net50, "net5.0")] - [InlineData(".NETCoreApp,Version=v6.0", RuntimeMoniker.Net60, "net6.0")] - [InlineData(".NETCoreApp,Version=v7.0", RuntimeMoniker.Net70, "net7.0")] - [InlineData(".NETCoreApp,Version=v8.0", RuntimeMoniker.Net80, "net8.0")] - [InlineData(".NETCoreApp,Version=v9.0", RuntimeMoniker.Net90, "net9.0")] - [InlineData(".NETCoreApp,Version=v10.0", RuntimeMoniker.Net10_0, "net10.0")] - [InlineData(".NETCoreApp,Version=v11.0", RuntimeMoniker.Net11_0, "net11.0")] - [InlineData(".NETCoreApp,Version=v123.0", RuntimeMoniker.NotRecognized, "net123.0")] - public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName, RuntimeMoniker expectedTfm, string expectedMsBuildMoniker) + [InlineData(".NETCoreApp,Version=v2.0", 2, 0, "netcoreapp2.0")] + [InlineData(".NETCoreApp,Version=v2.1", 2, 1, "netcoreapp2.1")] + [InlineData(".NETCoreApp,Version=v2.2", 2, 2, "netcoreapp2.2")] + [InlineData(".NETCoreApp,Version=v3.0", 3, 0, "netcoreapp3.0")] + [InlineData(".NETCoreApp,Version=v3.1", 3, 1, "netcoreapp3.1")] + [InlineData(".NETCoreApp,Version=v5.0", 5, 0, "net5.0")] + [InlineData(".NETCoreApp,Version=v6.0", 6, 0, "net6.0")] + [InlineData(".NETCoreApp,Version=v7.0", 7, 0, "net7.0")] + [InlineData(".NETCoreApp,Version=v8.0", 8, 0, "net8.0")] + [InlineData(".NETCoreApp,Version=v9.0", 9, 0, "net9.0")] + [InlineData(".NETCoreApp,Version=v10.0", 10, 0, "net10.0")] + [InlineData(".NETCoreApp,Version=v11.0", 11, 0, "net11.0")] + [InlineData(".NETCoreApp,Version=v123.0", 123, 0, "net123.0")] + public void TryGetVersionFromFrameworkNameHandlesValidInput(string frameworkName, int expectedMajor, int expectedMinor, string expectedTfm) { Assert.True(CoreRuntime.TryGetVersionFromFrameworkName(frameworkName, out var version)); - var runtime = CoreRuntime.FromVersion(version); + var runtime = CoreRuntime.From(version); - Assert.Equal(expectedTfm, runtime.RuntimeMoniker); - Assert.Equal(expectedMsBuildMoniker, runtime.MsBuildMoniker); + Assert.Equal(expectedMajor, runtime.Version.Major); + Assert.Equal(expectedMinor, runtime.Version.Minor); + Assert.Equal(expectedTfm, runtime.GetTfm()); } [Theory] @@ -42,20 +43,21 @@ public void TryGetVersionFromFrameworkNameHandlesInvalidInput(string? frameworkN } [Theory] - [InlineData(RuntimeMoniker.NetCoreApp21, "netcoreapp2.1", "Microsoft .NET Framework", "4.6.27817.01 @BuiltBy: dlab14-DDVSOWINAGE101 @Branch: release/2.1 @SrcCode: https://github.com/dotnet/coreclr/tree/6f78fbb3f964b4f407a2efb713a186384a167e5c")] - [InlineData(RuntimeMoniker.NetCoreApp22, "netcoreapp2.2", "Microsoft .NET Framework", "4.6.27817.03 @BuiltBy: dlab14-DDVSOWINAGE101 @Branch: release/2.2 @SrcCode: https://github.com/dotnet/coreclr/tree/ce1d090d33b400a25620c0145046471495067cc7")] - [InlineData(RuntimeMoniker.NetCoreApp30, "netcoreapp3.0", "Microsoft .NET Core", "3.0.0-preview8-28379-12")] - [InlineData(RuntimeMoniker.NetCoreApp31, "netcoreapp3.1", "Microsoft .NET Core", "3.1.0-something")] - [InlineData(RuntimeMoniker.Net50, "net5.0", "Microsoft .NET Core", "5.0.0-alpha1.19415.3")] - [InlineData(RuntimeMoniker.NotRecognized, "net123.0", "Microsoft .NET Core", "123.0.0-future")] - public void TryGetVersionFromProductInfoHandlesValidInput(RuntimeMoniker expectedTfm, string expectedMsBuildMoniker, string productName, string productVersion) + [InlineData(2, 1, "netcoreapp2.1", "Microsoft .NET Framework", "4.6.27817.01 @BuiltBy: dlab14-DDVSOWINAGE101 @Branch: release/2.1 @SrcCode: https://github.com/dotnet/coreclr/tree/6f78fbb3f964b4f407a2efb713a186384a167e5c")] + [InlineData(2, 2, "netcoreapp2.2", "Microsoft .NET Framework", "4.6.27817.03 @BuiltBy: dlab14-DDVSOWINAGE101 @Branch: release/2.2 @SrcCode: https://github.com/dotnet/coreclr/tree/ce1d090d33b400a25620c0145046471495067cc7")] + [InlineData(3, 0, "netcoreapp3.0", "Microsoft .NET Core", "3.0.0-preview8-28379-12")] + [InlineData(3, 1, "netcoreapp3.1", "Microsoft .NET Core", "3.1.0-something")] + [InlineData(5, 0, "net5.0", "Microsoft .NET Core", "5.0.0-alpha1.19415.3")] + [InlineData(123, 0, "net123.0", "Microsoft .NET Core", "123.0.0-future")] + public void TryGetVersionFromProductInfoHandlesValidInput(int expectedMajor, int expectedMinor, string expectedTfm, string productName, string productVersion) { Assert.True(CoreRuntime.TryGetVersionFromProductInfo(productVersion, productName, out var version)); - var runtime = CoreRuntime.FromVersion(version); + var runtime = CoreRuntime.From(version); - Assert.Equal(expectedTfm, runtime.RuntimeMoniker); - Assert.Equal(expectedMsBuildMoniker, runtime.MsBuildMoniker); + Assert.Equal(expectedMajor, runtime.Version.Major); + Assert.Equal(expectedMinor, runtime.Version.Minor); + Assert.Equal(expectedTfm, runtime.GetTfm()); } [Theory] @@ -72,24 +74,25 @@ public static IEnumerable FromNetCoreAppVersionHandlesValidInputArgume { string directoryPrefix = Path.GetTempPath(); // this test runs on Unix, it can not be hardcoded due to / \ difference - yield return new object[] { Path.Combine(directoryPrefix, "2.0.9") + Path.DirectorySeparatorChar, RuntimeMoniker.NetCoreApp20, "netcoreapp2.0" }; - yield return new object[] { Path.Combine(directoryPrefix, "2.1.12") + Path.DirectorySeparatorChar, RuntimeMoniker.NetCoreApp21, "netcoreapp2.1" }; - yield return new object[] { Path.Combine(directoryPrefix, "2.2.6") + Path.DirectorySeparatorChar, RuntimeMoniker.NetCoreApp22, "netcoreapp2.2" }; - yield return new object[] { Path.Combine(directoryPrefix, "3.0.0-preview8-28379-12") + Path.DirectorySeparatorChar, RuntimeMoniker.NetCoreApp30, "netcoreapp3.0" }; - yield return new object[] { Path.Combine(directoryPrefix, "5.0.0-alpha1.19422.13") + Path.DirectorySeparatorChar, RuntimeMoniker.Net50, "net5.0" }; - yield return new object[] { Path.Combine(directoryPrefix, "123.0.0") + Path.DirectorySeparatorChar, RuntimeMoniker.NotRecognized, "net123.0" }; + yield return new object[] { Path.Combine(directoryPrefix, "2.0.9") + Path.DirectorySeparatorChar, 2, 0, "netcoreapp2.0" }; + yield return new object[] { Path.Combine(directoryPrefix, "2.1.12") + Path.DirectorySeparatorChar, 2, 1, "netcoreapp2.1" }; + yield return new object[] { Path.Combine(directoryPrefix, "2.2.6") + Path.DirectorySeparatorChar, 2, 2, "netcoreapp2.2" }; + yield return new object[] { Path.Combine(directoryPrefix, "3.0.0-preview8-28379-12") + Path.DirectorySeparatorChar, 3, 0, "netcoreapp3.0" }; + yield return new object[] { Path.Combine(directoryPrefix, "5.0.0-alpha1.19422.13") + Path.DirectorySeparatorChar, 5, 0, "net5.0" }; + yield return new object[] { Path.Combine(directoryPrefix, "123.0.0") + Path.DirectorySeparatorChar, 123, 0, "net123.0" }; } [Theory] [MemberData(nameof(FromNetCoreAppVersionHandlesValidInputArguments))] - public void TryGetVersionFromRuntimeDirectoryHandlesValidInput(string runtimeDirectory, RuntimeMoniker expectedTfm, string expectedMsBuildMoniker) + public void TryGetVersionFromRuntimeDirectoryHandlesValidInput(string runtimeDirectory, int expectedMajor, int expectedMinor, string expectedTfm) { Assert.True(CoreRuntime.TryGetVersionFromRuntimeDirectory(runtimeDirectory, out var version)); - var runtime = CoreRuntime.FromVersion(version); + var runtime = CoreRuntime.From(version); - Assert.Equal(expectedTfm, runtime.RuntimeMoniker); - Assert.Equal(expectedMsBuildMoniker, runtime.MsBuildMoniker); + Assert.Equal(expectedMajor, runtime.Version.Major); + Assert.Equal(expectedMinor, runtime.Version.Minor); + Assert.Equal(expectedTfm, runtime.GetTfm()); } public static IEnumerable TryGetVersionFromRuntimeDirectoryInvalidInputArguments() @@ -114,13 +117,13 @@ public void CurrentRuntimeIsProperlyRecognized() #if NETFRAMEWORK if (OsDetector.IsWindows()) - Assert.True(runtime is ClrRuntime, $"Actual runtime: {runtime}, tfm: {runtime.MsBuildMoniker}, moniker: {runtime.RuntimeMoniker}"); + Assert.True(runtime is ClrRuntime, $"Actual runtime: {runtime}"); else - Assert.True(runtime is MonoRuntime, $"Actual runtime: {runtime}, tfm: {runtime.MsBuildMoniker}, moniker: {runtime.RuntimeMoniker}"); + Assert.True(runtime is MonoRuntime, $"Actual runtime: {runtime}"); #elif NET10_0_OR_GREATER - Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.Net10_0, $"Actual runtime: {runtime}, tfm: {runtime.MsBuildMoniker}, moniker: {runtime.RuntimeMoniker}"); + Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.Version.Major == 10, $"Actual runtime: {runtime}"); #else - Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.RuntimeMoniker == RuntimeMoniker.Net80, $"Actual runtime: {runtime}, tfm: {runtime.MsBuildMoniker}, moniker: {runtime.RuntimeMoniker}"); + Assert.True(runtime is CoreRuntime coreRuntime && coreRuntime.Version.Major == 8, $"Actual runtime: {runtime}"); #endif } } diff --git a/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs b/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs index 325cab5903..ce47c2d223 100644 --- a/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs +++ b/tests/BenchmarkDotNet.Tests/Serialization/DisassemblerModelSerializationTests.cs @@ -21,7 +21,7 @@ public void ClrMdArgsSerializationTest() printSource: true, maxDepth: 5, syntax: "Syntax", - tfm: "Tfm", + runtimeVersion: new Version(8, 0), filters: ["filter1", "filter2"], resultsPath: "/path/to/results" ); diff --git a/tests/BenchmarkDotNet.Tests/SettingsEqualityTests.cs b/tests/BenchmarkDotNet.Tests/SettingsEqualityTests.cs new file mode 100644 index 0000000000..7c9679ef20 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/SettingsEqualityTests.cs @@ -0,0 +1,91 @@ +using System; +using System.IO; +using BenchmarkDotNet.Toolchains.CoreRun; +using BenchmarkDotNet.Toolchains.Mono; +using BenchmarkDotNet.Toolchains.NativeAot; +using BenchmarkDotNet.Toolchains.R2R; +using BenchmarkDotNet.Toolchains.Wasm; + +namespace BenchmarkDotNet.Tests; + +// FileInfo/DirectoryInfo/Func members compare by reference by default. The *Settings records override equality to +// compare paths by value (FullName), so two settings built from distinct FileInfo instances pointing at the same path +// must be equal (and share a hash code) - otherwise toolchains built on them would defeat job dedup/build partitioning. +public class SettingsEqualityTests +{ + private const string PathA = "some/dir/tool.exe"; + private const string PathB = "some/other/tool.exe"; + + private static void AssertEqualWithHash(T x, T y) where T : notnull + { + Assert.Equal(x, y); + Assert.Equal(x.GetHashCode(), y.GetHashCode()); + } + + [Fact] + public void R2RSettings_ComparePacksByPath() + { + var a = new R2RSettings { Crossgen2Pack = new FileInfo(PathA) }; + var sameAsA = new R2RSettings { Crossgen2Pack = new FileInfo(PathA) }; + var differs = new R2RSettings { Crossgen2Pack = new FileInfo(PathB) }; + + AssertEqualWithHash(a, sameAsA); + Assert.NotEqual(a, differs); + } + + [Fact] + public void WasmSettings_CompareMainJsTemplateByPath_AndDefaultFormatterEqual() + { + var a = new WasmSettings { MainJsTemplate = new FileInfo(PathA) }; + var sameAsA = new WasmSettings { MainJsTemplate = new FileInfo(PathA) }; + var differs = new WasmSettings { MainJsTemplate = new FileInfo(PathB) }; + + AssertEqualWithHash(a, sameAsA); + Assert.NotEqual(a, differs); + // Two fresh instances use the same static default formatter, so they must be equal. + AssertEqualWithHash(new WasmSettings(), new WasmSettings()); + } + + [Fact] + public void CoreRunSettings_CompareSourceCoreRunByPath() + { + var a = new CoreRunSettings { SourceCoreRun = new FileInfo(PathA) }; + var sameAsA = new CoreRunSettings { SourceCoreRun = new FileInfo(PathA) }; + var differs = new CoreRunSettings { SourceCoreRun = new FileInfo(PathB) }; + + AssertEqualWithHash(a, sameAsA); + Assert.NotEqual(a, differs); + } + + [Fact] + public void LegacyMonoSettings_CompareMonoPathByPath() + { + var a = new MonoSettings { MonoPath = new FileInfo(PathA) }; + var sameAsA = new MonoSettings { MonoPath = new FileInfo(PathA) }; + var differs = new MonoSettings { MonoPath = new FileInfo(PathB) }; + + AssertEqualWithHash(a, sameAsA); + Assert.NotEqual(a, differs); + + // Derived AOT record chains into the base path comparison and adds its own AotArgs. + var aot = new MonoAotSettings { MonoPath = new FileInfo(PathA), AotArgs = "--aot=full" }; + var aotSame = new MonoAotSettings { MonoPath = new FileInfo(PathA), AotArgs = "--aot=full" }; + var aotDiffers = new MonoAotSettings { MonoPath = new FileInfo(PathA), AotArgs = "--aot" }; + + AssertEqualWithHash(aot, aotSame); + Assert.NotEqual(aot, aotDiffers); + } + + [Fact] + public void NativeAotSettings_CompareLocalIlcPackagesByPath() + { + // WithLocalBuild requires the directory to exist, so use the test output directory. + var existing = AppContext.BaseDirectory; + + var a = NativeAotSettings.Default.WithLocalBuild(new DirectoryInfo(existing)); + var sameAsA = NativeAotSettings.Default.WithLocalBuild(new DirectoryInfo(existing)); + + AssertEqualWithHash(a, sameAsA); + Assert.NotEqual(a, NativeAotSettings.Default); + } +} diff --git a/tests/BenchmarkDotNet.Tests/Shared/JobToolchainExtensions.cs b/tests/BenchmarkDotNet.Tests/Shared/JobToolchainExtensions.cs new file mode 100644 index 0000000000..b021414a47 --- /dev/null +++ b/tests/BenchmarkDotNet.Tests/Shared/JobToolchainExtensions.cs @@ -0,0 +1,29 @@ +using BenchmarkDotNet.Configs; +using BenchmarkDotNet.Jobs; +using BenchmarkDotNet.Parameters; +using BenchmarkDotNet.Running; +using BenchmarkDotNet.Tests.Mocks; + +namespace BenchmarkDotNet.Toolchains; + +/// +/// Test-only helper that resolves the toolchain for a without a real benchmark case. The core +/// only resolves toolchains per BenchmarkCase (which carries the full context); tests use this to inspect +/// the toolchain a parsed config job resolves to, or to get the default job's toolchain as a test parameter. +/// +internal static class JobToolchainExtensions +{ + // GetDefaultToolchain needs a BenchmarkCase (some runtimes inspect the descriptor's target assembly); tests have + // no real benchmark here, so we wrap the job in one built from a mock descriptor. It's not a LinqPad assembly and + // targets the same TFM as the test host, so it resolves to the same toolchain a bare job used to. + private static readonly Descriptor MockDescriptor = new(MockFactory.MockType, MockFactory.MockMethodInfo); + + internal static IToolchain GetToolchain(this Job job) + { + if (job.Infrastructure.TryGetToolchain(out var toolchain)) + return toolchain; + + var benchmarkCase = BenchmarkCase.Create(MockDescriptor, job, ParameterInstances.Empty, ManualConfig.CreateEmpty().CreateImmutableConfig()); + return benchmarkCase.GetRuntime().GetDefaultToolchain(benchmarkCase); + } +} diff --git a/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockFactory.cs b/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockFactory.cs index 5db6e9f6c1..1d9514a13b 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockFactory.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/Mocks/MockFactory.cs @@ -62,6 +62,18 @@ public static Summary CreateSummaryWithBiasedDistribution(Type benchmarkType, in []); } + public static Summary CreateSummary(Type benchmarkType, IConfig config) => new Summary( + "MockSummary", + BenchmarkConverter.TypeToBenchmarks(benchmarkType, config).BenchmarksCases.Select(CreateSimpleReport).ToImmutableArray(), + new HostEnvironmentInfoBuilder().WithoutDotNetSdkVersion().Build(), + string.Empty, + string.Empty, + TimeSpan.FromMinutes(1), + config.CultureInfo ?? DefaultCultureInfo.Instance, + [], + [], + config.SummaryStyle); + public static Summary CreateSummary(IConfig config) => new Summary( "MockSummary", CreateReports(config), diff --git a/tests/BenchmarkDotNet.Tests/Shared/Mocks/Toolchain/MockToolchain.cs b/tests/BenchmarkDotNet.Tests/Shared/Mocks/Toolchain/MockToolchain.cs index 5649e36c66..38554b961a 100644 --- a/tests/BenchmarkDotNet.Tests/Shared/Mocks/Toolchain/MockToolchain.cs +++ b/tests/BenchmarkDotNet.Tests/Shared/Mocks/Toolchain/MockToolchain.cs @@ -1,4 +1,5 @@ using BenchmarkDotNet.Characteristics; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Reports; using BenchmarkDotNet.Running; @@ -11,7 +12,7 @@ namespace BenchmarkDotNet.Tests.Mocks.Toolchain { public class MockToolchain(Func> measurer) : IToolchain { - public string Name => nameof(MockToolchain); + public Runtime Runtime => UnknownRuntime.Instance; public IGenerator Generator => new MockGenerator(); public IBuilder Builder => new MockBuilder(); public IExecutor Executor { get; private set; } = new MockExecutor(measurer); diff --git a/tests/BenchmarkDotNet.Tests/TargetFrameworkVersionParsingTestscs.cs b/tests/BenchmarkDotNet.Tests/TargetFrameworkVersionParsingTestscs.cs deleted file mode 100644 index a08f7aef40..0000000000 --- a/tests/BenchmarkDotNet.Tests/TargetFrameworkVersionParsingTestscs.cs +++ /dev/null @@ -1,21 +0,0 @@ -using BenchmarkDotNet.Disassemblers; - -namespace BenchmarkDotNet.Tests -{ - public class TargetFrameworkVersionParsingTestscs - { - [Theory] - [InlineData("net461", 4, 6, 1)] - [InlineData("net48", 4, 8, -1)] - [InlineData("net7.0", 7, 0, -1)] - [InlineData("net7.0-windows8", 7, 0, -1)] - public void RuntimeVersionCanBeParsedFromTfm(string tfm, int major, int minor, int build) - { - Version version = State.ParseVersion(tfm); - - Assert.Equal(major, version.Major); - Assert.Equal(minor, version.Minor); - Assert.Equal(build, version.Build); - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs b/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs deleted file mode 100644 index 133d85ea4d..0000000000 --- a/tests/BenchmarkDotNet.Tests/Validators/RuntimeValidatorTests.cs +++ /dev/null @@ -1,123 +0,0 @@ -using BenchmarkDotNet.Attributes; -using BenchmarkDotNet.Configs; -using BenchmarkDotNet.Environments; -using BenchmarkDotNet.Jobs; -using BenchmarkDotNet.Running; -using BenchmarkDotNet.Toolchains.CsProj; -using BenchmarkDotNet.Validators; - -namespace BenchmarkDotNet.Tests.Validators; - -public class RuntimeValidatorTests -{ - [Fact] - public async Task SameRuntime_Should_Success() - { - // Arrange - var config = new TestConfig1().CreateImmutableConfig(); - var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); - var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); - - // Act - var errors = await RuntimeValidator.DontFailOnError.ValidateAsync(parameters).Select(e => e.Message).ToArrayAsync(); - - // Assert - Assert.Empty(errors); - } - - [Fact] - public async Task NullRuntimeMixed_Should_Failed() - { - // Arrange - var config = new TestConfig2().CreateImmutableConfig(); - var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); - var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); - - // Act - var errors = await RuntimeValidator.DontFailOnError.ValidateAsync(parameters).Select(e => e.Message).ToArrayAsync(); - - // Assert - { - var expectedMessage = "Job(Dry) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; - Assert.Contains(expectedMessage, errors); - } - { - var expectedMessage = "Job(Toolchain=.NET 11.0) doesn't have a Runtime characteristic. It's recommended to specify runtime by using WithRuntime explicitly."; - Assert.Contains(expectedMessage, errors); - } - } - - [Fact] - public async Task NotNullRuntimeOnly_Should_Success() - { - // Arrange - var config = new TestConfig3().CreateImmutableConfig(); - var runInfo = BenchmarkConverter.TypeToBenchmarks(typeof(DummyBenchmark), config); - var parameters = new ValidationParameters(runInfo.BenchmarksCases, config); - - // Act - var errors = await RuntimeValidator.DontFailOnError.ValidateAsync(parameters).Select(e => e.Message).ToArrayAsync(); - - // Assert - Assert.Empty(errors); - } - - public class DummyBenchmark - { - [Benchmark] - public void Benchmark() - { - } - } - - // TestConfig that expicitly specify runtime. - private class TestConfig1 : ManualConfig - { - public TestConfig1() - { - var baseJob = Job.Dry; - - WithOption(ConfigOptions.DisableOptimizationsValidator, true); - AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); - - AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); - AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90)); - } - } - - // TestConfig that contains job that don't specify runtime. - private class TestConfig2 : ManualConfig - { - public TestConfig2() - { - var baseJob = Job.Dry; - - WithOption(ConfigOptions.DisableOptimizationsValidator, true); - AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); - - AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80)); - AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) - .WithRuntime(CoreRuntime.Core90)); - - // Validate error message for auto generated jobid. - AddJob(Job.Default.WithToolchain(CsProjCoreToolchain.NetCoreApp11_0)); - } - } - - // TestConfig that expicitly specify runtime. - private class TestConfig3 : ManualConfig - { - public TestConfig3() - { - var baseJob = Job.Dry; - - WithOption(ConfigOptions.DisableOptimizationsValidator, true); - AddColumnProvider(DefaultConfig.Instance.GetColumnProviders().ToArray()); - - AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp80) - .WithRuntime(CoreRuntime.Core80)); ; - AddJob(baseJob.WithToolchain(CsProjCoreToolchain.NetCoreApp90) - .WithRuntime(CoreRuntime.Core90)); - } - } -} diff --git a/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs index e9e3a18195..9a7056de84 100644 --- a/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs +++ b/tests/BenchmarkDotNet.Tests/dotMemory/DotMemoryTests.cs @@ -1,15 +1,20 @@ using BenchmarkDotNet.Diagnostics.dotMemory; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using System.Reflection; namespace BenchmarkDotNet.Tests.dotMemory; public class DotMemoryTests { [Fact] - public void AllRuntimeMonikerAreKnown() + public void AllRuntimeMonikersAreKnown() { var diagnoser = new DotMemoryDiagnoser(); - foreach (RuntimeMoniker moniker in Enum.GetValues()) - diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions + foreach (var field in typeof(RuntimeMoniker).GetFields(BindingFlags.Public | BindingFlags.Static)) + { + var moniker = (string)field.GetValue(null)!; + diagnoser.IsSupported(Runtime.Parse(moniker)); // Just check that it doesn't throw exceptions + } } -} \ No newline at end of file +} diff --git a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs index 906500271b..074c78dae3 100644 --- a/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs +++ b/tests/BenchmarkDotNet.Tests/dotTrace/DotTraceTests.cs @@ -1,15 +1,20 @@ using BenchmarkDotNet.Diagnostics.dotTrace; +using BenchmarkDotNet.Environments; using BenchmarkDotNet.Jobs; +using System.Reflection; namespace BenchmarkDotNet.Tests.dotTrace; public class DotTraceTests { [Fact] - public void AllRuntimeMonikerAreKnown() + public void AllRuntimeMonikersAreKnown() { var diagnoser = new DotTraceDiagnoser(); - foreach (RuntimeMoniker moniker in Enum.GetValues()) - diagnoser.IsSupported(moniker); // Just check that it doesn't throw exceptions + foreach (var field in typeof(RuntimeMoniker).GetFields(BindingFlags.Public | BindingFlags.Static)) + { + var moniker = (string)field.GetValue(null)!; + diagnoser.IsSupported(Runtime.Parse(moniker)); // Just check that it doesn't throw exceptions + } } -} \ No newline at end of file +}