diff --git a/src/BenchmarkDotNet/Extensions/XDocumentExtensions.cs b/src/BenchmarkDotNet/Extensions/XDocumentExtensions.cs new file mode 100644 index 0000000000..d8b3497757 --- /dev/null +++ b/src/BenchmarkDotNet/Extensions/XDocumentExtensions.cs @@ -0,0 +1,17 @@ +#if NETSTANDARD2_0 +using System.Xml.Linq; + +namespace BenchmarkDotNet.Extensions; + +internal static class XDocumentExtensions +{ + /// + /// Helper extension method for netstandard2.0. + /// It provides an API `SaveAsync`, but the actual processing is performed synchronously. + /// + public static async ValueTask SaveAsync(this XDocument doc, Stream stream, SaveOptions options, CancellationToken cancellationToken) + { + doc.Save(stream, options); + } +} +#endif diff --git a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs index 9a744e6ee1..35ab2d9368 100644 --- a/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs +++ b/src/BenchmarkDotNet/Running/BenchmarkRunnerClean.cs @@ -103,14 +103,7 @@ private static async ValueTask RunCore(BenchmarkRunInfo[] benchmarkRu var buildPartitions = BenchmarkPartitioner.CreateForBuild(supportedBenchmarks, resolver); eventProcessor.OnStartBuildStage(buildPartitions); - var sequentialBuildPartitions = buildPartitions.Where(partition => - partition.Benchmarks.Any(x => x.Config.Options.IsSet(ConfigOptions.DisableParallelBuild)) - // .Net SDK 8+ supports ArtifactsPath for proper parallel builds. - // Older SDKs may produce builds with incorrect bindings if more than 1 partition is built in parallel. - || (partition.RepresentativeBenchmarkCase.GetToolchain().Generator is DotNetCliGenerator - && partition.RepresentativeBenchmarkCase.GetRuntime().RuntimeMoniker.GetRuntimeVersion().Major < 8) - ) - .ToArray(); + var sequentialBuildPartitions = buildPartitions.Where(partition => partition.Benchmarks.Any(x => x.Config.Options.IsSet(ConfigOptions.DisableParallelBuild))).ToArray(); var parallelBuildPartitions = buildPartitions.Except(sequentialBuildPartitions).ToArray(); Dictionary buildResults = parallelBuildPartitions.Length > 0 diff --git a/src/BenchmarkDotNet/Templates/BenchmarkDotNet.Build.props.txt b/src/BenchmarkDotNet/Templates/BenchmarkDotNet.Build.props.txt new file mode 100644 index 0000000000..bfe8a19491 --- /dev/null +++ b/src/BenchmarkDotNet/Templates/BenchmarkDotNet.Build.props.txt @@ -0,0 +1,11 @@ + + + + + + + + $OUTPUT_PATH$ + $PUBLISH_DIR$ + + diff --git a/src/BenchmarkDotNet/Templates/BenchmarkDotNet.Build.targets.txt b/src/BenchmarkDotNet/Templates/BenchmarkDotNet.Build.targets.txt new file mode 100644 index 0000000000..4ae78eb8ad --- /dev/null +++ b/src/BenchmarkDotNet/Templates/BenchmarkDotNet.Build.targets.txt @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/BenchmarkDotNet/Templates/CsProj.txt b/src/BenchmarkDotNet/Templates/CsProj.txt index 9afef8cf5e..2add1180ed 100644 --- a/src/BenchmarkDotNet/Templates/CsProj.txt +++ b/src/BenchmarkDotNet/Templates/CsProj.txt @@ -1,4 +1,5 @@ + $PROGRAMNAME$ @@ -36,4 +37,5 @@ $RUNTIMESETTINGS$ + diff --git a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt index f005cd82b4..10fbf761fc 100644 --- a/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt +++ b/src/BenchmarkDotNet/Templates/MonoAOTLLVMCsProj.txt @@ -1,4 +1,6 @@ - + + + $CSPROJPATH$ $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Mono.props')) @@ -19,7 +21,6 @@ false $PROGRAMNAME$ false - false true BenchmarkDotNet.Autogenerated.UniqueProgramName true @@ -73,7 +74,6 @@ - + diff --git a/src/BenchmarkDotNet/Templates/R2RCsProj.txt b/src/BenchmarkDotNet/Templates/R2RCsProj.txt index e881418b1b..9dc424cd22 100644 --- a/src/BenchmarkDotNet/Templates/R2RCsProj.txt +++ b/src/BenchmarkDotNet/Templates/R2RCsProj.txt @@ -1,4 +1,5 @@ + @@ -16,8 +17,6 @@ false BenchmarkDotNet.Autogenerated.UniqueProgramName - - false @@ -61,4 +60,6 @@ + + diff --git a/src/BenchmarkDotNet/Templates/WasmCsProj.txt b/src/BenchmarkDotNet/Templates/WasmCsProj.txt index 68915fe4f6..37072f47bf 100644 --- a/src/BenchmarkDotNet/Templates/WasmCsProj.txt +++ b/src/BenchmarkDotNet/Templates/WasmCsProj.txt @@ -1,4 +1,6 @@ - + + + $CSPROJPATH$ $([System.IO.Path]::ChangeExtension('$(OriginalCSProjPath)', '.Wasm.props')) @@ -23,10 +25,17 @@ $CORECLR_OVERRIDES$ true $RUN_AOT$ $(RunAOTCompilation) - false false false BenchmarkDotNet.Autogenerated.UniqueProgramName + + + false + + + + + false @@ -60,4 +69,5 @@ $CORECLR_OVERRIDES$ + diff --git a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs index 08956ba6ea..a0d197e3f4 100644 --- a/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/CsProj/CsProjGenerator.cs @@ -19,6 +19,8 @@ namespace BenchmarkDotNet.Toolchains.CsProj public class CsProjGenerator : DotNetCliGenerator, IEquatable { private const string DefaultSdkName = "Microsoft.NET.Sdk"; + internal const string DllGathererProjectName = "DllGatherer.csproj"; + internal const string AutogeneratedProjectName = "BenchmarkDotNet.Autogenerated.csproj"; private static readonly ImmutableArray SettingsWeWantToCopy = new[] { @@ -68,11 +70,40 @@ protected override string GetBuildArtifactsDirectoryPath(BuildPartition buildPar } protected override string GetProjectFilePath(string buildArtifactsDirectoryPath) - => Path.Combine(buildArtifactsDirectoryPath, "BenchmarkDotNet.Autogenerated.csproj"); + => Path.Combine(buildArtifactsDirectoryPath, AutogeneratedProjectName); protected override string GetBinariesDirectoryPath(string buildArtifactsDirectoryPath, string configuration) => Path.Combine(buildArtifactsDirectoryPath, "bin", configuration, TargetFrameworkMoniker); + protected override async ValueTask GenerateCustomBuildHooksAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) + { + var benchmark = buildPartition.RepresentativeBenchmarkCase; + var benchmarkProjectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); + + var buildArtifactsDirectoryPath = artifactsPaths.BuildArtifactsDirectoryPath; + + // Trim tfm part from binariesDirectoryPath on IntegrationTests. (When ArtifactsPath is not used, it's appended automatically) + var binariesDirectoryPath = buildPartition.ForcedNoDependenciesForIntegrationTests + ? artifactsPaths.BinariesDirectoryPath.Substring(0, artifactsPaths.BinariesDirectoryPath.Length - TargetFrameworkMoniker.Length - 1) + : artifactsPaths.BinariesDirectoryPath; + + var propsPath = Path.Combine(buildArtifactsDirectoryPath, "BenchmarkDotNet.Build.props"); + var props = GetTemplateContent("BenchmarkDotNet.Build.props.txt"); + await File.WriteAllTextAsync(propsPath, props.ToString(), cancellationToken).ConfigureAwait(false); + + var targetsPath = Path.Combine(buildArtifactsDirectoryPath, "BenchmarkDotNet.Build.targets"); + var targets = GetTemplateContent("BenchmarkDotNet.Build.targets.txt"); + await File.WriteAllTextAsync(targetsPath, targets.ToString(), cancellationToken).ConfigureAwait(false); + + string GetTemplateContent(string templateName) + => new StringBuilder(ResourceHelper.LoadTemplate(templateName)) + .Replace("$DLLGATHERER_PROJ$", DllGathererProjectName) + .Replace("$AUTOGENERATED_PROJ$", AutogeneratedProjectName) + .Replace("$OUTPUT_PATH$", binariesDirectoryPath) + .Replace("$PUBLISH_DIR$", artifactsPaths.PublishDirectoryPath) + .ToString(); + } + protected override ValueTask GenerateBuildScriptAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, CancellationToken cancellationToken) { string projectFilePath = GetProjectFilePath(buildPartition.RepresentativeBenchmarkCase.Descriptor.Type, NullLogger.Instance).FullName; @@ -89,6 +120,7 @@ protected override ValueTask GenerateBuildScriptAsync(BuildPartition buildPartit protected override async ValueTask GenerateProjectAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) { + // Generate Autogenerated project file. await File.WriteAllTextAsync( artifactsPaths.ProjectFilePath, await GenerateBuildProject(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false), @@ -97,35 +129,15 @@ await GenerateBuildProject(buildPartition, artifactsPaths, logger, cancellationT .ConfigureAwait(false); // Integration tests are built without dependencies, so we skip gathering dlls. - if (!buildPartition.ForcedNoDependenciesForIntegrationTests) - { - await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); - } - } - - private async ValueTask GenerateBuildProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) - { - var benchmark = buildPartition.RepresentativeBenchmarkCase; - var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); - - var xmlDoc = new XmlDocument(); - xmlDoc.Load(projectFile.FullName); - var (customProperties, sdkName) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); + if (buildPartition.ForcedNoDependenciesForIntegrationTests) + return; - return new StringBuilder(await ResourceHelper.LoadTemplateAsync("CsProj.txt", cancellationToken).ConfigureAwait(false)) - .Replace("$PLATFORM$", buildPartition.Platform.ToConfig()) - .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) - .Replace("$CSPROJPATH$", projectFile.FullName) - .Replace("$TFM$", TargetFrameworkMoniker) - .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) - .Replace("$RUNTIMESETTINGS$", GetRuntimeSettings(benchmark.Job.Environment.Gc, buildPartition.Resolver)) - .Replace("$COPIEDSETTINGS$", customProperties) - .Replace("$SDKNAME$", sdkName) - .ToString(); + // Gather DLL references and modify Autogenerated project file + await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); } private static string GetDllGathererPath(string filePath) - => Path.Combine(Path.GetDirectoryName(filePath)!, $"DllGatherer{Path.GetExtension(filePath)}"); + => Path.Combine(Path.GetDirectoryName(filePath)!, $"{Path.GetFileNameWithoutExtension(DllGathererProjectName)}{Path.GetExtension(filePath)}"); protected async ValueTask GatherReferencesAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) { @@ -141,11 +153,7 @@ protected async ValueTask GatherReferencesAsync(BuildPartition buildPartition, A using (var gathererStream = File.Create(gathererProject)) { -#if NETSTANDARD2_0 - doc.Save(gathererStream, SaveOptions.None); -#else await doc.SaveAsync(gathererStream, SaveOptions.None, cancellationToken).ConfigureAwait(false); -#endif } await File.WriteAllTextAsync(emptyMainFile, """ @@ -186,13 +194,16 @@ public static int Main(string[] args) return; } + // Gets bin directory of DllGather project from output file that generated by custom MSBuild target. + var binDirectory = File.ReadAllText(Path.Combine(artifactsPaths.BuildArtifactsDirectoryPath, "DllGatherer.txt")).Trim(); + // Delete the dll from the gatherer project to prevent duplicate references. - File.Delete(Path.Combine(artifactsPaths.BinariesDirectoryPath, $"{artifactsPaths.ProgramName}.dll")); + File.Delete(Path.Combine(binDirectory, $"{artifactsPaths.ProgramName}.dll")); doc = XDocument.Load(artifactsPaths.ProjectFilePath); var itemGroup = new XElement("ItemGroup"); doc.Root!.Add(itemGroup); - foreach (var assemblyFile in Directory.GetFiles(artifactsPaths.BinariesDirectoryPath, "*.dll")) + foreach (var assemblyFile in Directory.GetFiles(binDirectory, "*.dll")) { itemGroup.Add(new XElement("Reference", new XAttribute("Include", Path.GetFileNameWithoutExtension(assemblyFile)), @@ -202,11 +213,28 @@ public static int Main(string[] args) } using var projectStream = File.Create(artifactsPaths.ProjectFilePath); -#if NETSTANDARD2_0 - doc.Save(projectStream, SaveOptions.None); -#else await doc.SaveAsync(projectStream, SaveOptions.None, cancellationToken).ConfigureAwait(false); -#endif + } + + private async ValueTask GenerateBuildProject(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) + { + var benchmark = buildPartition.RepresentativeBenchmarkCase; + var projectFile = GetProjectFilePath(benchmark.Descriptor.Type, logger); + + var xmlDoc = new XmlDocument(); + xmlDoc.Load(projectFile.FullName); + var (customProperties, sdkName) = GetSettingsThatNeedToBeCopied(xmlDoc, projectFile); + + return new StringBuilder(await ResourceHelper.LoadTemplateAsync("CsProj.txt", cancellationToken).ConfigureAwait(false)) + .Replace("$PLATFORM$", buildPartition.Platform.ToConfig()) + .Replace("$CODEFILENAME$", Path.GetFileName(artifactsPaths.ProgramCodePath)) + .Replace("$CSPROJPATH$", projectFile.FullName) + .Replace("$TFM$", TargetFrameworkMoniker) + .Replace("$PROGRAMNAME$", artifactsPaths.ProgramName) + .Replace("$RUNTIMESETTINGS$", GetRuntimeSettings(benchmark.Job.Environment.Gc, buildPartition.Resolver)) + .Replace("$COPIEDSETTINGS$", customProperties) + .Replace("$SDKNAME$", sdkName) + .ToString(); } /// diff --git a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs index 01005e450f..d7d2c33833 100644 --- a/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs +++ b/src/BenchmarkDotNet/Toolchains/DotNetCli/DotNetCliCommand.cs @@ -4,6 +4,7 @@ using BenchmarkDotNet.Loggers; using BenchmarkDotNet.Portability; using BenchmarkDotNet.Running; +using BenchmarkDotNet.Toolchains.CsProj; using BenchmarkDotNet.Toolchains.Results; using JetBrains.Annotations; using System.Text; @@ -53,6 +54,9 @@ public DotNetCliCommand WithArguments(string arguments) public DotNetCliCommand WithCliPath(string cliPath) => new(cliPath, FilePath, TargetFrameworkMoniker, Arguments, GenerateResult, Logger, BuildPartition, EnvironmentVariables, Timeout, LogOutput); + private bool UseNoDependencies + => BuildPartition.IsCustomBuildConfiguration || Path.GetFileName(FilePath) == CsProjGenerator.AutogeneratedProjectName; + [PublicAPI] public async Task RestoreThenBuildAsync(CancellationToken cancellationToken = default) { @@ -67,32 +71,29 @@ public async Task RestoreThenBuildAsync(CancellationToken cancellat return result.ToBuildResult(GenerateResult); } - // On our CI, Integration tests take too much time, because each benchmark run rebuilds BenchmarkDotNet itself. - // To reduce the total duration of the CI workflows, we build all the projects without dependencies - if (BuildPartition.ForcedNoDependenciesForIntegrationTests) + // Run `dotnet restore` and `dotnet build` command. + if (UseNoDependencies) { - var restoreResult = await DotNetCliCommandExecutor.ExecuteAsync( - WithArguments(GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, $"{Arguments} --no-dependencies", "restore-no-deps", excludeOutput: true)), - cancellationToken).ConfigureAwait(false); + // On our CI, Integration tests take too much time, because each benchmark run rebuilds BenchmarkDotNet itself. + // To reduce the total duration of the CI workflows, we build all the projects without dependencies + + // Run `dotnet restore` command with `--no-dependency`. + var restoreResult = await RestoreNoDependenciesAsync(cancellationToken).ConfigureAwait(false); if (!restoreResult.IsSuccess) return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - var result = await DotNetCliCommandExecutor.ExecuteAsync( - WithArguments( - GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, TargetFrameworkMoniker, $"{Arguments} --no-restore --no-dependencies", "build-no-restore-no-deps", excludeOutput: true) - ), - cancellationToken).ConfigureAwait(false); - - return result.ToBuildResult(GenerateResult); + // Run `dotnet build` command with `--no-restore --no-dependency`. + var buildResult = await BuildNoRestoreNoDependenciesAsync(cancellationToken).ConfigureAwait(false); + return buildResult.ToBuildResult(GenerateResult); } else { + // Run `dotnet restore` command. var restoreResult = await RestoreAsync(cancellationToken).ConfigureAwait(false); if (!restoreResult.IsSuccess) return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); - // We no longer retry with --no-dependencies, because it fails with --output set at the same time, - // and the artifactsPaths.BinariesDirectoryPath is set before we try to build, so we cannot overwrite it. + // Run `dotnet build` command with `--no-restore` command. var result = await BuildNoRestoreAsync(cancellationToken).ConfigureAwait(false); return result.ToBuildResult(GenerateResult); } @@ -112,13 +113,38 @@ public async Task RestoreThenBuildThenPublishAsync(CancellationToke return result.ToBuildResult(GenerateResult); } - var restoreResult = await RestoreAsync(cancellationToken).ConfigureAwait(false); - if (!restoreResult.IsSuccess) - return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); + if (UseNoDependencies) + { + // Run `dotnet restore` command with `--no-dependencies`. + var restoreResult = await RestoreNoDependenciesAsync(cancellationToken).ConfigureAwait(false); + if (!restoreResult.IsSuccess) + return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); + + // Run `dotnet build` command with `--no-restore --no-dependencies`. + var buildResult = await BuildNoRestoreNoDependenciesAsync(cancellationToken).ConfigureAwait(false); + if (!buildResult.IsSuccess) + return BuildResult.Failure(GenerateResult, buildResult.AllInformation); + + // Run `dotnet publish` command with `--no-build`. + var publishResult = await PublishNoBuildAsync(cancellationToken).ConfigureAwait(false); + return publishResult.ToBuildResult(GenerateResult); + } + else + { + // Run `dotnet restore` command. + var restoreResult = await RestoreAsync(cancellationToken).ConfigureAwait(false); + if (!restoreResult.IsSuccess) + return BuildResult.Failure(GenerateResult, restoreResult.AllInformation); + + // Run `dotnet build` command with `--no-restore`. + var buildResult = await BuildNoRestoreAsync(cancellationToken).ConfigureAwait(false); + if (!buildResult.IsSuccess) + return BuildResult.Failure(GenerateResult, buildResult.AllInformation); - // We use the implicit build in the publish command. We stopped doing a separate build step because we set the --output. - var publishResult = await PublishNoRestoreAsync(cancellationToken).ConfigureAwait(false); - return publishResult.ToBuildResult(GenerateResult); + // Run `dotnet publish` command with `--no-build`. + var publishResult = await PublishNoBuildAsync(cancellationToken).ConfigureAwait(false); + return publishResult.ToBuildResult(GenerateResult); + } } public Task RestoreAsync(CancellationToken cancellationToken = default) @@ -126,6 +152,11 @@ public Task RestoreAsync(CancellationToken cancellationT WithArguments(GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, Arguments, "restore")), cancellationToken); + public Task RestoreNoDependenciesAsync(CancellationToken cancellationToken = default) + => DotNetCliCommandExecutor.ExecuteAsync( + WithArguments(GetRestoreCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, $"{Arguments} --no-dependencies", "restore-no-deps")), + cancellationToken); + public Task BuildAsync(CancellationToken cancellationToken = default) => DotNetCliCommandExecutor.ExecuteAsync( WithArguments(GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, TargetFrameworkMoniker, Arguments, "build")), @@ -136,64 +167,81 @@ public Task BuildNoRestoreAsync(CancellationToken cancel WithArguments(GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, TargetFrameworkMoniker, $"{Arguments} --no-restore", "build-no-restore")), cancellationToken); + public Task BuildNoRestoreNoDependenciesAsync(CancellationToken cancellationToken = default) + => DotNetCliCommandExecutor.ExecuteAsync( + WithArguments(GetBuildCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, TargetFrameworkMoniker, $"{Arguments} --no-restore --no-dependencies", "build-no-restore-no-deps")), + cancellationToken); + public Task PublishAsync(CancellationToken cancellationToken = default) => DotNetCliCommandExecutor.ExecuteAsync( WithArguments(GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, TargetFrameworkMoniker, Arguments, "publish")), cancellationToken); - // PublishNoBuildAndNoRestore was removed because we set --output in the build step. We use the implicit build included in the publish command. - public Task PublishNoRestoreAsync(CancellationToken cancellationToken = default) + public Task PublishNoBuildAsync(CancellationToken cancellationToken = default) => DotNetCliCommandExecutor.ExecuteAsync( - WithArguments(GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, TargetFrameworkMoniker, $"{Arguments} --no-restore", "publish-no-restore")), + WithArguments(GetPublishCommand(GenerateResult.ArtifactsPaths, BuildPartition, FilePath, TargetFrameworkMoniker, $"{Arguments} --no-build", "publish-no-build")), cancellationToken); - internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string filePath, string? extraArguments = null, string? binLogSuffix = null, bool excludeOutput = false) - => new StringBuilder() + internal static string GetRestoreCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string filePath, string? extraArguments = null, string? binLogSuffix = null) + => new StringBuilder(256) .AppendArgument("restore") - .AppendArgument($"\"{filePath}\"") + .AppendArgument(filePath.ToRelativePath(artifactsPaths).QuoteIfNeeded()) // restore doesn't support -f argument. - .AppendArgument(artifactsPaths.PackagesDirectoryName.IsBlank() ? string.Empty : $"--packages \"{artifactsPaths.PackagesDirectoryName}\"") + .AppendArgument(GetArtifactsPathArguments(buildPartition)) + .AppendArgument(artifactsPaths.PackagesDirectoryName.IsBlank() ? string.Empty : $"--packages {artifactsPaths.PackagesDirectoryName.QuoteIfNeeded()}") .AppendArgument(GetCustomMsBuildArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)) .AppendArgument(extraArguments) .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) - .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) - .MaybeAppendOutputPaths(artifactsPaths, true, excludeOutput) + .AppendArgument(GetMsBuildBinLogArgument(buildPartition, filePath, binLogSuffix)) .ToString(); - internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string filePath, string tfm, string? extraArguments = null, string? binLogSuffix = null, bool excludeOutput = false) - => new StringBuilder() + internal static string GetBuildCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string filePath, string tfm, string? extraArguments = null, string? binLogSuffix = null) + => new StringBuilder(256) .AppendArgument("build") - .AppendArgument($"\"{filePath}\"") + .AppendArgument(filePath.ToRelativePath(artifactsPaths).QuoteIfNeeded()) .AppendArgument($"-f {tfm}") .AppendArgument($"-c {buildPartition.BuildConfiguration}") + .AppendArgument(GetArtifactsPathArguments(buildPartition)) .AppendArgument(GetCustomMsBuildArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)) .AppendArgument(extraArguments) .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) - .AppendArgument(artifactsPaths.PackagesDirectoryName.IsBlank() ? string.Empty : $"/p:NuGetPackageRoot=\"{artifactsPaths.PackagesDirectoryName}\"") - .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) - .MaybeAppendOutputPaths(artifactsPaths, excludeOutput: excludeOutput) + .AppendArgument(artifactsPaths.PackagesDirectoryName.IsBlank() ? string.Empty : $"/p:NuGetPackageRoot={artifactsPaths.PackagesDirectoryName.QuoteIfNeeded()}") + .AppendArgument(GetMsBuildBinLogArgument(buildPartition, filePath, binLogSuffix)) .ToString(); internal static string GetPublishCommand(ArtifactsPaths artifactsPaths, BuildPartition buildPartition, string filePath, string tfm, string? extraArguments = null, string? binLogSuffix = null) - => new StringBuilder() + => new StringBuilder(256) .AppendArgument("publish") - .AppendArgument($"\"{filePath}\"") + .AppendArgument(filePath.ToRelativePath(artifactsPaths).QuoteIfNeeded()) .AppendArgument($"-f {tfm}") .AppendArgument($"-c {buildPartition.BuildConfiguration}") + .AppendArgument(GetArtifactsPathArguments(buildPartition)) .AppendArgument(GetCustomMsBuildArguments(buildPartition.RepresentativeBenchmarkCase, buildPartition.Resolver)) .AppendArgument(extraArguments) .AppendArgument(GetMandatoryMsBuildSettings(buildPartition.BuildConfiguration)) - .AppendArgument(artifactsPaths.PackagesDirectoryName.IsBlank() ? string.Empty : $"/p:NuGetPackageRoot=\"{artifactsPaths.PackagesDirectoryName}\"") - .AppendArgument(GetMsBuildBinLogArgument(buildPartition, binLogSuffix)) - .MaybeAppendOutputPaths(artifactsPaths) + .AppendArgument(artifactsPaths.PackagesDirectoryName.IsBlank() ? string.Empty : $"/p:NuGetPackageRoot={artifactsPaths.PackagesDirectoryName.QuoteIfNeeded()}") + .AppendArgument(GetMsBuildBinLogArgument(buildPartition, filePath, binLogSuffix)) .ToString(); - private static string GetMsBuildBinLogArgument(BuildPartition buildPartition, string? suffix) + private static string GetArtifactsPathArguments(BuildPartition buildPartition) + { + // Don't use `--artifacts-path` for integration tests. Because it build with `no-dependencies`. + if (buildPartition.ForcedNoDependenciesForIntegrationTests) + return ""; + + var artifactsPath = ".artifacts"; + return $"--artifacts-path {artifactsPath}"; + } + + private static string GetMsBuildBinLogArgument(BuildPartition buildPartition, string projectPath, string? suffix) { if (!buildPartition.GenerateMSBuildBinLog || suffix.IsBlank()) return string.Empty; - return $"\"-bl:{buildPartition.ProgramName}-{suffix}.binlog\""; + var projectName = Path.GetFileNameWithoutExtension(projectPath); + + var fileName = $"{projectName}-{suffix}.binlog".QuoteIfNeeded(); + return $"-bl:{fileName}"; } private static string GetCustomMsBuildArguments(BenchmarkCase benchmarkCase, IResolver resolver) @@ -222,24 +270,55 @@ private static string GetMandatoryMsBuildSettings(string buildConfiguration) } } - internal static class DotNetCliCommandExtensions + file static class DotNetCliCommandExtensions { - // Fix #1377 (see comments in #1773). - // We force the project to output binaries to a new directory. - // Specifying --output and --no-dependencies breaks the build (because the previous build was not done using the custom output path), - // so we don't include it if we're building no-deps (only supported for integration tests). - internal static StringBuilder MaybeAppendOutputPaths(this StringBuilder stringBuilder, ArtifactsPaths artifactsPaths, bool isRestore = false, bool excludeOutput = false) - => excludeOutput - ? stringBuilder - : stringBuilder - // Use AltDirectorySeparatorChar so it's not interpreted as an escaped quote `\"`. - // Use a subdirectory for ArtifactsPath so that DefaultItemExcludes (which the SDK - // sets to $(ArtifactsPath)/**) doesn't cover project-level files like wwwroot/. - .AppendArgument($"/p:ArtifactsPath=\"{artifactsPaths.BuildArtifactsDirectoryPath}{Path.AltDirectorySeparatorChar}.artifacts{Path.AltDirectorySeparatorChar}\"") - .AppendArgument($"/p:OutDir=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") - // OutputPath is legacy, per-project version of OutDir. We set both just in case. https://github.com/dotnet/msbuild/issues/87 - .AppendArgument($"/p:OutputPath=\"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\"") - .AppendArgument($"/p:PublishDir=\"{artifactsPaths.PublishDirectoryPath}{Path.AltDirectorySeparatorChar}\"") - .AppendArgument(isRestore ? string.Empty : $"--output \"{artifactsPaths.BinariesDirectoryPath}{Path.AltDirectorySeparatorChar}\""); + internal static string ToRelativePath(this string path, ArtifactsPaths artifactsPaths) + { + var buildArtifactsDirectoryPath = $"{artifactsPaths.BuildArtifactsDirectoryPath}{Path.DirectorySeparatorChar}"; + if (path.StartsWith(buildArtifactsDirectoryPath)) + return path.Substring(buildArtifactsDirectoryPath.Length); + + return path; + } + + internal static string QuoteIfNeeded(this string commandArg) + { + ArgumentNullException.ThrowIfNull(commandArg); + + if (commandArg.Length == 0) + return "\"\""; + + if (!commandArg.Any(char.IsWhiteSpace) && !commandArg.Contains('"')) + return commandArg; + + var builder = new StringBuilder(commandArg.Length + 2); + builder.Append('"'); + + var backslashCount = 0; + foreach (var c in commandArg) + { + switch (c) + { + case '\\': + backslashCount++; + continue; + case '"': + builder.Append('\\', backslashCount * 2 + 1); + builder.Append('"'); + break; + default: + builder.Append('\\', backslashCount); + builder.Append(c); + break; + } + + backslashCount = 0; + } + + builder.Append('\\', backslashCount * 2); + builder.Append('"'); + + return builder.ToString(); + } } } diff --git a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs index 177d0985af..8183c79dfb 100644 --- a/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs +++ b/src/BenchmarkDotNet/Toolchains/GeneratorBase.cs @@ -34,6 +34,7 @@ public async ValueTask GenerateProjectAsync(BuildPartition build // There is no async file copy API, so we just do it synchronously. We are likely on a ThreadPool thread here anyway if this generator is ran in parallel. CopyAllRequiredFiles(artifactsPaths); + await GenerateCustomBuildHooksAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); await GenerateCodeAsync(buildPartition, artifactsPaths, cancellationToken).ConfigureAwait(false); await GenerateAppConfigAsync(buildPartition, artifactsPaths, cancellationToken).ConfigureAwait(false); await GenerateNuGetConfigAsync(artifactsPaths, cancellationToken).ConfigureAwait(false); @@ -101,6 +102,11 @@ [PublicAPI] protected virtual void CopyAllRequiredFiles(ArtifactsPaths artifacts /// [PublicAPI] protected virtual ValueTask GenerateProjectAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) => new(); + /// + /// generates .props and .targets files to customize build. + /// + [PublicAPI] protected virtual ValueTask GenerateCustomBuildHooksAsync(BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger, CancellationToken cancellationToken) => new(); + /// /// generates a script can be used when debugging compilation issues /// diff --git a/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs b/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs index 543283bd0c..2de05e173d 100644 --- a/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/Mono/MonoGenerator.cs @@ -12,13 +12,10 @@ public MonoGenerator(string targetFrameworkMoniker, string cliPath, string packa protected override string GetRuntimeSettings(GcMode gcMode, IResolver resolver) { - // Workaround for following issues. - // 1. 'Found multiple publish output files with the same relative path' error - // 2. NU1102 error occurs when passing /p:UseMonoRuntime=true to the dotnet cli with projects containing .NET 9.0 or higher. #3000 + // NU1102 error occurs when passing /p:UseMonoRuntime=true to the dotnet cli with projects containing .NET 9.0 or higher. #3000 return base.GetRuntimeSettings(gcMode, resolver) + """ - false true """; diff --git a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs index 0253e4fc85..03cf712600 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoAotLLVM/MonoAotLLVMGenerator.cs @@ -52,6 +52,10 @@ protected override async ValueTask GenerateProjectAsync(BuildPartition buildPart await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); + // Integration tests are built without dependencies, so we skip gathering dlls. + if (buildPartition.ForcedNoDependenciesForIntegrationTests) + return; + await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); } diff --git a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs index 2692eb6f1b..2e7d74fb7f 100644 --- a/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/MonoWasm/WasmGenerator.cs @@ -87,6 +87,10 @@ protected async ValueTask GenerateProjectFileAsync(BuildPartition buildPartition await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); + // Integration tests are built without dependencies, so we skip gathering dlls. + if (buildPartition.ForcedNoDependenciesForIntegrationTests) + return; + await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); } diff --git a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs index 7ce4b32a5e..a2c725628e 100644 --- a/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs +++ b/src/BenchmarkDotNet/Toolchains/NativeAot/Generator.cs @@ -131,12 +131,19 @@ protected override async ValueTask GenerateProjectAsync(BuildPartition buildPart await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, GenerateProjectForNuGetBuild(projectFile, buildPartition, artifactsPaths, logger), cancellationToken).ConfigureAwait(false); - await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); + // Generate `bdn_generated.rd.xml` await GenerateReflectionFileAsync(artifactsPaths, cancellationToken).ConfigureAwait(false); + + // Integration tests are built without dependencies, so we skip gathering dlls. + if (buildPartition.ForcedNoDependenciesForIntegrationTests) + return; + + await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); } private string GenerateProjectForNuGetBuild(string projectFilePath, BuildPartition buildPartition, ArtifactsPaths artifactsPaths, ILogger logger) => $""" + Exe {TargetFrameworkMoniker} @@ -157,7 +164,6 @@ private string GenerateProjectForNuGetBuild(string projectFilePath, BuildPartiti {ilcGenerateStackTraceData} {ilcGenerateStackTraceData} false - false false {GetInstructionSetSettings(buildPartition)} @@ -177,6 +183,7 @@ private string GenerateProjectForNuGetBuild(string projectFilePath, BuildPartiti latest + """; diff --git a/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs b/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs index 506d22ac33..8c0ba96402 100644 --- a/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs +++ b/src/BenchmarkDotNet/Toolchains/R2R/R2RGenerator.cs @@ -47,6 +47,10 @@ protected override async ValueTask GenerateProjectAsync(BuildPartition buildPart await File.WriteAllTextAsync(artifactsPaths.ProjectFilePath, content, cancellationToken).ConfigureAwait(false); + // Integration tests are built without dependencies, so we skip gathering dlls. + if (buildPartition.ForcedNoDependenciesForIntegrationTests) + return; + await GatherReferencesAsync(buildPartition, artifactsPaths, logger, cancellationToken).ConfigureAwait(false); } diff --git a/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs index 369265cb92..4a994e484c 100644 --- a/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs +++ b/src/BenchmarkDotNet/Validators/DotNetSdkValidator.cs @@ -1,5 +1,6 @@ using BenchmarkDotNet.Environments; using BenchmarkDotNet.Extensions; +using BenchmarkDotNet.Helpers; using BenchmarkDotNet.Jobs; using BenchmarkDotNet.Running; using System.ComponentModel; @@ -20,10 +21,19 @@ public static IEnumerable ValidateCoreSdks(string? customDotNet yield return cliPathError; yield break; } + var requiredSdkVersion = benchmark.GetRuntime().RuntimeMoniker.GetRuntimeVersion(); if (!GetInstalledDotNetSdks(customDotNetCliPath).Any(sdk => sdk >= requiredSdkVersion)) { yield return new ValidationError(true, $"The required .NET Core SDK version {requiredSdkVersion} or higher for runtime moniker {benchmark.Job.Environment.Runtime!.RuntimeMoniker} is not installed.", benchmark); + yield break; + } + + // Validate actual .NET SDK version. (.NET 8 SDK is minimum requirement to use ArtifactsPath) + if (TryGetDotNetSdkVersion(customDotNetCliPath, out string rawVersionText, out Version? sdkVersion)) + { + if (sdkVersion.Major < 8) + yield return new ValidationError(true, $"The .NET 8 SDK is the minimum requirement for building the project. Resolved SDK version: {rawVersionText}", benchmark); } } @@ -182,5 +192,22 @@ private static string CheckFor45PlusVersion(int releaseKey) return ""; } + + private static bool TryGetDotNetSdkVersion( + string? customDotNetCliPath, + out string rawSdkVersion, + [NotNullWhen(true)] out Version? sdkVersion) + { + string exePath = customDotNetCliPath.IsBlank() + ? "dotnet" + : customDotNetCliPath!; + + rawSdkVersion = ProcessHelper.RunAndReadOutput(exePath, "--version") ?? ""; + if (Version.TryParse(CoreRuntime.GetParsableVersionPart(rawSdkVersion), out sdkVersion)) + return true; + + sdkVersion = null; + return false; + } } } \ No newline at end of file diff --git a/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs b/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs index 8e37f112dd..dc4a9a5127 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/CancellationTokenTests.cs @@ -66,7 +66,7 @@ public void BenchmarkWithCancellationTokenProperty_ReceivesToken_Wasm(string jav .AddJob(Job.Dry .WithRuntime(new WasmRuntime(dotnetVersion, RuntimeMoniker.WasmNet10_0, "wasm", false, javaScriptEngine)) .WithToolchain(WasmToolchain.From(netCoreAppSettings))) - .WithBuildTimeout(TimeSpan.FromSeconds(240)) + .WithBuildTimeout(TimeSpan.FromSeconds(480)) .WithOption(ConfigOptions.LogBuildOutput, true) .WithOption(ConfigOptions.GenerateMSBuildBinLog, false); @@ -133,7 +133,7 @@ public async Task RunWithCancellationTokenIsCancelled_Wasm(string javaScriptEngi .WithRuntime(new WasmRuntime(dotnetVersion, RuntimeMoniker.WasmNet10_0, "wasm", false, javaScriptEngine)) .WithToolchain(WasmToolchain.From(netCoreAppSettings))) .AddDiagnoser(diagnoser) - .WithBuildTimeout(TimeSpan.FromSeconds(240)) + .WithBuildTimeout(TimeSpan.FromSeconds(480)) .WithOption(ConfigOptions.LogBuildOutput, true) .WithOption(ConfigOptions.GenerateMSBuildBinLog, false); diff --git a/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs index f9cc6fef84..4fc7a4cb5e 100644 --- a/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs +++ b/tests/BenchmarkDotNet.IntegrationTests/WakeLockTests.cs @@ -13,7 +13,7 @@ public class WakeLockTests : BenchmarkTestExecutor { private const string PingEventName = @"Global\WakeLockTests-ping"; private const string PongEventName = @"Global\WakeLockTests-pong"; - private static readonly TimeSpan testTimeout = TimeSpan.FromMinutes(1); + private static readonly TimeSpan testTimeout = TimeSpan.FromMinutes(5); // Set 5 minutes for running tests with `ForcedNoDependenciesForIntegrationTests=false` private readonly OutputLogger logger; public WakeLockTests(ITestOutputHelper output) : base(output)