diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/InternalAPI/InternalAPI.Unshipped.txt index 464e563731..49d476e64c 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/InternalAPI/InternalAPI.Unshipped.txt @@ -11,6 +11,7 @@ static Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsSummaryReporter *REMOVED*Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsSummaryReporter.AzureDevOpsSummaryReporter(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Configurations.IConfiguration! configuration, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory) -> void *REMOVED*abstract Microsoft.Testing.Extensions.SlowTestReporterBase.EmitSlowTestAsync(string! testName, System.TimeSpan elapsed, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsClient.CreateHttpClientHandler() -> System.Net.Http.HttpClientHandler! static Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsClient.ShouldOptInToAutomaticDecompression(System.Net.Http.HttpClientHandler! handler) -> bool Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsPublisher.AzureDevOpsTestResultsPublisher(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Configurations.IConfiguration! configuration, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Services.ITestApplicationProcessExitCode! testApplicationProcessExitCode, Microsoft.Testing.Extensions.AzureDevOpsReport.IAzureDevOpsTestResultsClient! client, Microsoft.Testing.Platform.Helpers.ITask! task, Microsoft.Testing.Platform.Helpers.IClock! clock, Microsoft.Testing.Platform.Logging.ILogger! logger, Microsoft.Testing.Extensions.AzureDevOpsReport.AzureDevOpsTestResultsPublisherOptions! options) -> void diff --git a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt index 1c3f69be1f..ed4dba7318 100644 --- a/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.CtrfReport/InternalAPI/InternalAPI.Unshipped.txt @@ -27,6 +27,7 @@ static Microsoft.Testing.Extensions.CtrfReport.CtrfReportMerger.MergeToFileAsync static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Platform.Services.ArtifactNamingHelper.ResolveAndSanitize(string! template, string! processName, string! processId, System.DateTimeOffset timestamp, System.Func! sanitizeLeafFileName) -> string! Microsoft.Testing.Extensions.MergeOutputFileHelper static Microsoft.Testing.Extensions.MergeOutputFileHelper.BuildCaseFoldedProbePath(string! directory, string! probeFileName) -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs index fac35b631b..fa10973db0 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/GitHubActionsExitCode.cs @@ -60,6 +60,7 @@ public static string GetReason(int exitCode) (int)ExitCode.IncompatibleProtocolVersion => GitHubActionsResources.ExitCodeReasonIncompatibleProtocolVersion, (int)ExitCode.TestExecutionStoppedForMaxFailedTests => GitHubActionsResources.ExitCodeReasonTestExecutionStoppedForMaxFailedTests, (int)ExitCode.CoverageThresholdFailed => GitHubActionsResources.ExitCodeReasonCoverageThresholdFailed, + (int)ExitCode.TestExecutionStoppedAtDeadline => GitHubActionsResources.ExitCodeReasonTestExecutionStoppedAtDeadline, _ => GitHubActionsResources.ExitCodeReasonUnknown, }; } diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt index fe414f9ae6..459329f601 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/InternalAPI/InternalAPI.Unshipped.txt @@ -28,6 +28,7 @@ static Microsoft.Testing.Extensions.TestNodeIdentity.GetDisplayLabel(Microsoft.T *REMOVED*static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSlowTestReporter.BuildNoticeLine(string! testName, System.TimeSpan elapsed) -> string! *REMOVED*static Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSummaryReporter.BuildMarkdown(System.Collections.Generic.IReadOnlyList! records, string! assemblyName, string! targetFrameworkMoniker) -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSourceLocation Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSourceLocation.GitHubActionsSourceLocation() -> void Microsoft.Testing.Extensions.GitHubActionsReport.GitHubActionsSourceLocation.GitHubActionsSourceLocation(string! relativeNormalizedPath, int lineNumber) -> void diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md index 2146f04449..55ff2b5452 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/PACKAGE.md @@ -15,12 +15,12 @@ dotnet add package Microsoft.Testing.Extensions.GitHubActionsReport This package extends Microsoft.Testing.Platform with: - **Per-assembly log groups**: emits `::group::` / `::endgroup::` workflow commands so each test assembly's output is collapsed by default in the runner UI -- **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter. Skipped tests are surfaced as `::warning` annotations so they are visible in the Annotations tab too. When the test session completes with a non-test-result failure — a `--minimum-expected-tests` violation, a run that discovered zero tests, a `--maximum-failed-tests` stop, or a test-adapter session failure — a single run-level `::error` is emitted describing the [Microsoft.Testing.Platform exit code](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes) +- **Failure annotations**: emits an `::error` workflow command for each failing test so failures appear in the workflow Annotations tab and, when the source location can be resolved, on the pull request's "Files changed" diff gutter. Skipped tests are surfaced as `::warning` annotations so they are visible in the Annotations tab too. When the test session completes with a non-test-result failure — a `--minimum-expected-tests` violation, a run that discovered zero tests, a `--maximum-failed-tests` stop, a deadline-triggered early stop, or a test-adapter session failure — a single run-level `::error` is emitted describing the [Microsoft.Testing.Platform exit code](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-troubleshooting#exit-codes) - **Job summary**: writes one markdown roll-up (totals, failures, slowest tests) to the file pointed to by `GITHUB_STEP_SUMMARY`, which GitHub renders on the workflow run summary page. With an SDK that supports required artifact post-processing, multi-module `dotnet test` runs produce one authoritative overall section using the SDK's outer duration and exit verdict, with deterministic per-assembly details underneath. Older SDKs preserve the per-assembly sections. A non-test-result failure exit code is called out so a failure is not hidden behind a green ✅ - **Slow-test notices**: emits a `::notice` workflow command for any test still running past a threshold (default 60 seconds) > [!NOTE] -> The exit-code callout and run-level annotation only cover outcomes the extension can observe once the in-process test session has finished. Those are: `ZeroTests` (8), `MinimumExpectedTestsPolicyViolation` (9), `TestAdapterTestSessionFailure` (10), and `TestExecutionStoppedForMaxFailedTests` (13). `AtLeastOneTestFailed` (2) is already conveyed by the per-test failures, so it gets no separate callout. A hard abort/cancellation (`TestSessionAborted`, 3) short-circuits end-of-session reporting, and codes raised before or after the session — e.g. `InvalidCommandLine` (5) or `TestHostProcessExitedNonGracefully` (7) — occur outside the extension's reach, so none of those are surfaced here. +> The exit-code callout and run-level annotation only cover outcomes the extension can observe once the in-process test session has finished. Those are: `ZeroTests` (8), `MinimumExpectedTestsPolicyViolation` (9), `TestAdapterTestSessionFailure` (10), `TestExecutionStoppedForMaxFailedTests` (13), and `TestExecutionStoppedAtDeadline` (15). `AtLeastOneTestFailed` (2) is already conveyed by the per-test failures, so it gets no separate callout. A hard abort/cancellation (`TestSessionAborted`, 3) short-circuits end-of-session reporting, and codes raised before or after the session — e.g. `InvalidCommandLine` (5) or `TestHostProcessExitedNonGracefully` (7) — occur outside the extension's reach, so none of those are surfaced here. > > Cross-module aggregation is negotiated with `dotnet test`. If the SDK does not provide the authoritative run-summary context, the extension keeps its standalone behavior; a manually invoked post-processor labels totals as observed and leaves overall duration and exit verdict unavailable rather than reconstructing them. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx index f1344c17ff..cf8c44612f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/GitHubActionsResources.resx @@ -181,6 +181,9 @@ One or more code coverage thresholds were not met. + + Test execution was stopped early because a CI-imposed deadline was approaching. + The test run reported a non-success exit code. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf index fe5cfdfd7c..a8fcc11b48 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.cs.xlf @@ -82,6 +82,11 @@ Testovací adaptér oznámil selhání testovací relace. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. Spouštění testů se zastavilo po dosažení limitu nastaveného parametrem --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf index 794ca5d1a5..e150c96e93 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.de.xlf @@ -82,6 +82,11 @@ Der Testadapter hat einen Fehler in der Testsitzung gemeldet. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. Die Testausführung wurde beendet, nachdem das von --maximum-failed-tests festgelegte Limit erreicht wurde. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf index d59e4e6dc2..08c1d07d82 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.es.xlf @@ -82,6 +82,11 @@ El adaptador de prueba notificó un error de sesión de prueba. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. La ejecución de pruebas se detuvo al alcanzar el límite establecido por --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf index 442c74ad18..48dc5cf683 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.fr.xlf @@ -82,6 +82,11 @@ L’adaptateur de test a signalé l’échec d’une session de test. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. L’exécution de tests s’est arrêtée après avoir atteint la limite définie par --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf index 68bd97fc1f..ec7e2ea00e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.it.xlf @@ -82,6 +82,11 @@ L'adattatore di test ha segnalato un errore della sessione di test. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. L'esecuzione dei test è stata interrotta dopo aver raggiunto il limite impostato da --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf index 71fce27175..63d6764e88 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ja.xlf @@ -82,6 +82,11 @@ テスト アダプターから、テスト セッションの失敗が報告されました。 + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. --maximum-failed-tests で設定された上限に達したため、テストの実行が停止しました。 diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf index e04185c076..ddd20b96a3 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ko.xlf @@ -82,6 +82,11 @@ 테스트 어댑터에서 테스트 세션 실패를 보고했습니다. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. --maximum-failed-tests에서 설정한 한도에 도달해 테스트 실행이 중지되었습니다. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf index af591d62e3..c786b22cf7 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pl.xlf @@ -82,6 +82,11 @@ Adapter testowy zgłosił błąd sesji testowej. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. Wykonanie testów zostało zatrzymane po osiągnięciu limitu ustawionego za pomocą opcji --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf index 7e42f807a2..8adb341669 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.pt-BR.xlf @@ -82,6 +82,11 @@ O adaptador de teste relatou uma falha na sessão de teste. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. A execução de teste foi interrompida após atingir o limite definido por --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf index 869010128b..09bfdc93cf 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.ru.xlf @@ -82,6 +82,11 @@ Адаптер тестирования сообщил о сбое тестового сеанса. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. Выполнение тестов остановлено после достижения предела, заданного параметром --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf index 38abcfe6e9..8df88a4ed0 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.tr.xlf @@ -82,6 +82,11 @@ Test bağdaştırıcısı bir test oturumu hatası bildirdi. + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. --maximum-failed-tests tarafından belirlenen sınıra ulaşıldıktan sonra test yürütme durduruldu. diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf index 797769695a..4dcf1e94c5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hans.xlf @@ -82,6 +82,11 @@ 测试适配器报告了测试会话失败。 + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. 测试执行在达到 --maximum-failed-tests 设置的限制后停止。 diff --git a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf index 3b9e75e7ac..c231502106 100644 --- a/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.GitHubActionsReport/Resources/xlf/GitHubActionsResources.zh-Hant.xlf @@ -82,6 +82,11 @@ 測試配接器報告測試工作階段失敗。 + + Test execution was stopped early because a CI-imposed deadline was approaching. + Test execution was stopped early because a CI-imposed deadline was approaching. + + Test execution stopped after reaching the limit set by --maximum-failed-tests. 測試執行在達到 --maximum-failed-tests 所設定的限制之後停止。 diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/HangDumpProcessLifetimeHandler.cs b/src/Platform/Microsoft.Testing.Extensions.HangDump/HangDumpProcessLifetimeHandler.cs index d632969bff..f58086016c 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/HangDumpProcessLifetimeHandler.cs +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/HangDumpProcessLifetimeHandler.cs @@ -48,8 +48,38 @@ internal sealed class HangDumpProcessLifetimeHandler : ITestHostProcessLifetimeH private readonly ManualResetEventSlim _waitConsumerPipeName = new(false); private readonly List _dumpFiles = []; + // Guards the "take the dump only once" gate (_dumpTaken) together with publishing the running + // dump task (_activityIndicatorTask), so disposal always observes and awaits the winning dump. +#if NET9_0_OR_GREATER + private readonly Lock _dumpLock = new(); +#else + private readonly object _dumpLock = new(); +#endif + private TimeSpan? _activityTimerValue; private Timer? _activityTimer; + private DateTimeOffset? _deadlineDumpAt; + private Timer? _deadlineTimer; + + /// + /// throws for due times above ~49.7 days (its internal limit is + /// milliseconds). A deadline that far out is effectively "never" + /// for a test run, so we clamp to this maximum instead of throwing during setup. + /// + private static readonly TimeSpan MaxTimerDueTime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + + /// + /// Upper bound for the optional in-progress-test query before taking a dump. A connected but + /// wedged host never answers the request/reply, and the application token is not cancelled while + /// the run is still "in progress" (which is exactly when the deadline dump fires), so an unbounded + /// query would block the dump and kill indefinitely and consume the whole dump margin. The query + /// is issued once per dump of the tree, not once per process, so this is the total worst case for + /// the whole tree and stays a small slice of the default 30s dump margin however many processes + /// are dumped; the healthy path answers in milliseconds. + /// + private static readonly TimeSpan InProgressTestsQueryTimeout = TimeSpan.FromSeconds(5); + + private int _dumpTaken; private Task? _waitConnectionTask; private Task? _activityIndicatorTask; private NamedPipeServer? _singleConnectionNamedPipeServer; @@ -58,6 +88,12 @@ internal sealed class HangDumpProcessLifetimeHandler : ITestHostProcessLifetimeH private ITestHostProcessInformation? _testHostProcessInformation; private NamedPipeClient? _namedPipeClient; + // Cancels the pipe handshake in OnTestHostProcessStartedAsync when a dump wins the race. Read and + // written under _dumpLock, and null whenever no handshake is in flight. Cancelled outside the lock, + // because cancellation runs the waiters' continuations inline and those continue into the handshake, + // which takes _dumpLock again on its way out. + private CancellationTokenSource? _handshakeCancellationTokenSource; + public HangDumpProcessLifetimeHandler( PipeNameDescription pipeNameDescription, IMessageBus messageBus, @@ -129,6 +165,15 @@ await _outputDisplay.DisplayAsync( await _logger.LogInformationAsync($"Hang dump timeout setup {_activityTimerValue}.").ConfigureAwait(false); + // In addition to the inactivity timeout above, honor an absolute CI deadline (if provided). + // We compute the wall-clock instant at which we should start taking the dump so that the dump + // has a chance to complete before the CI runner hard-kills the process. + if (DeadlineHelper.TryGetDeadline(_environment, out DateTimeOffset deadline)) + { + _deadlineDumpAt = DeadlineHelper.SubtractSaturating(deadline, DeadlineHelper.GetDumpMargin(_environment)); + await _logger.LogInformationAsync($"Hang dump deadline setup {_deadlineDumpAt:o}.").ConfigureAwait(false); + } + _singleConnectionNamedPipeServer = new(_pipeNameDescription, CallbackAsync, _environment, _logger, _task, cancellationToken); _singleConnectionNamedPipeServer.RegisterSerializer(new VoidResponseSerializer(), typeof(VoidResponse)); _singleConnectionNamedPipeServer.RegisterSerializer(new ConsumerPipeNameRequestSerializer(), typeof(ConsumerPipeNameRequest)); @@ -147,7 +192,15 @@ private async Task CallbackAsync(IRequest request) if (request is ConsumerPipeNameRequest consumerPipeNameRequest) { await _logger.LogDebugAsync($"Consumer pipe name received '{consumerPipeNameRequest.PipeName}'").ConfigureAwait(false); - _namedPipeClient = new NamedPipeClient(consumerPipeNameRequest.PipeName, _environment); + + // exitProcessOnConnectionLoss: false, because this is an auxiliary channel. It carries nothing but + // the best-effort in-progress-test query used to annotate a dump, and the peer is a test host we + // are often about to dump and kill -- so a disconnect here is expected rather than fatal. With the + // default (true) a host that drops while the query is in flight would call IEnvironment.Exit on + // this controller, killing the very process that still has to take and publish the dump, and the + // catch in QueryInProgressTestsWithTimeoutAsync would never run. Surfacing it as an exception lets + // the query fall back to an empty list and the dump continue. + _namedPipeClient = new NamedPipeClient(consumerPipeNameRequest.PipeName, _environment, exitProcessOnConnectionLoss: false); _namedPipeClient.RegisterSerializer(new GetInProgressTestsResponseSerializer(), typeof(GetInProgressTestsResponse)); _namedPipeClient.RegisterSerializer(new GetInProgressTestsRequestSerializer(), typeof(GetInProgressTestsRequest)); _namedPipeClient.RegisterSerializer(new VoidResponseSerializer(), typeof(VoidResponse)); @@ -175,22 +228,98 @@ public async Task OnTestHostProcessStartedAsync(ITestHostProcessInformation test ApplicationStateGuard.Ensure(_waitConnectionTask is not null); ApplicationStateGuard.Ensure(_singleConnectionNamedPipeServer is not null); + // Read the pipe server once, here, where the guard above proves it is set. The dereference is in a + // nested try several awaits down, and a field can in principle change under those awaits, so a local + // is what makes the null-safety contract hold at the point it is actually used. + NamedPipeServer singleConnectionNamedPipeServer = _singleConnectionNamedPipeServer; + _testHostProcessInformation = testHostProcessInformation; - await _logger.LogDebugAsync($"Wait for test host connection to the server pipe '{_singleConnectionNamedPipeServer.PipeName.Name}'").ConfigureAwait(false); - await _waitConnectionTask.TimeoutAfterAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false); - using CancellationTokenSource timeout = new(TimeoutHelper.DefaultHangTimeSpanTimeout); - using var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeout.Token); - _waitConsumerPipeName.Wait(linkedCancellationToken.Token); - ApplicationStateGuard.Ensure(_namedPipeClient is not null); - await _namedPipeClient.ConnectAsync(cancellationToken).TimeoutAfterAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false); - await _logger.LogDebugAsync($"Connected to the test host server pipe '{_namedPipeClient.PipeName}'").ConfigureAwait(false); - - _activityTimer = new Timer( - _ => _activityIndicatorTask = TakeDumpOfTreeAsync(cancellationToken), - null, - _activityTimerValue!.Value, - TimeSpan.FromMilliseconds(-1)); + // The pipe handshake below must be interruptible by a dump. Killing the test host does not + // complete this process's own WaitConnectionAsync -- that pipe is waiting for a client that will + // now never connect -- so without this token a host that wedged before connecting keeps Started + // blocked for DefaultHangTimeSpanTimeout (five minutes), far past the default 30s dump margin and + // the CI hard deadline, and OnTestHostProcessExitedAsync never runs to publish the dump that was + // taken. Published before the deadline timer is armed so a timer that fires immediately (a + // deadline already in the past) still finds it. + var handshakeCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + lock (_dumpLock) + { + _handshakeCancellationTokenSource = handshakeCancellation; + } + + CancellationToken handshakeToken = handshakeCancellation.Token; + + try + { + // Arm the absolute CI deadline timer as early as possible, before we block on the pipe + // handshake below. If the test host wedges during startup (never connects back over the + // pipe), those waits would otherwise block well past the deadline and the deadline dump/kill + // would never be armed, which defeats the purpose of the deadline. The dump path only needs + // the test host PID, which we already have here; the in-progress-test list (which needs the + // consumer pipe) is best-effort and skipped when the pipe never connected. + if (_deadlineDumpAt is { } deadlineDumpAt) + { + TimeSpan dueTime = deadlineDumpAt - _clock.UtcNow; + if (dueTime < TimeSpan.Zero) + { + dueTime = TimeSpan.Zero; + } + else if (dueTime > MaxTimerDueTime) + { + // Clamp far-future deadlines so the Timer ctor does not throw. The run (and this + // timer) is disposed long before the clamped due time elapses, so it never fires early. + dueTime = MaxTimerDueTime; + } + + _deadlineTimer = new Timer( + _ => TriggerDumpOnce(cancellationToken, triggeredByDeadline: true), + null, + dueTime, + TimeSpan.FromMilliseconds(-1)); + } + + // Once a dump has started, the test host is being dumped and killed out from under this + // handshake, so the pipe waits below will throw (cancellation, timeout, or a torn-down pipe). + // Let Started return normally in that case so the lifetime handler still receives + // OnTestHostProcessExitedAsync, which is where the dump files are published; otherwise a + // deadline dump would be taken but never surfaced as an artifact. + try + { + await _logger.LogDebugAsync($"Wait for test host connection to the server pipe '{singleConnectionNamedPipeServer.PipeName.Name}'").ConfigureAwait(false); + await _waitConnectionTask.TimeoutAfterAsync(TimeoutHelper.DefaultHangTimeSpanTimeout, handshakeToken).ConfigureAwait(false); + using CancellationTokenSource timeout = new(TimeoutHelper.DefaultHangTimeSpanTimeout); + using var linkedCancellationToken = CancellationTokenSource.CreateLinkedTokenSource(handshakeToken, timeout.Token); + _waitConsumerPipeName.Wait(linkedCancellationToken.Token); + ApplicationStateGuard.Ensure(_namedPipeClient is not null); + await _namedPipeClient.ConnectAsync(handshakeToken).TimeoutAfterAsync(TimeoutHelper.DefaultHangTimeSpanTimeout, handshakeToken).ConfigureAwait(false); + await _logger.LogDebugAsync($"Connected to the test host server pipe '{_namedPipeClient.PipeName}'").ConfigureAwait(false); + + // The inactivity timer only makes sense once the host has connected and can send activity + // signals; before that there is nothing to reset it. The deadline timer above is independent. + _activityTimer = new Timer( + _ => TriggerDumpOnce(cancellationToken, triggeredByDeadline: false), + null, + _activityTimerValue!.Value, + TimeSpan.FromMilliseconds(-1)); + } + catch (Exception ex) when (Volatile.Read(ref _dumpTaken) != 0) + { + // A dump is already in progress; the failed handshake is expected. Return normally so + // OnTestHostProcessExitedAsync runs and publishes the dump that is being taken. + await _logger.LogDebugAsync($"Test host handshake failed after the dump started; continuing so the dump can be published. {ex}").ConfigureAwait(false); + } + } + finally + { + // Unpublish before disposing, so a later dump reads null instead of a disposed source. + lock (_dumpLock) + { + _handshakeCancellationTokenSource = null; + } + + handshakeCancellation.Dispose(); + } } private static string GetDiskInfo() @@ -225,6 +354,15 @@ public async Task OnTestHostProcessExitedAsync(ITestHostProcessInformation testH #endif } + if (_deadlineTimer is not null) + { +#if NETCOREAPP + await _deadlineTimer.DisposeAsync().ConfigureAwait(false); +#else + _deadlineTimer.Dispose(); +#endif + } + if (!testHostProcessInformation.HasExitedGracefully) { _logger.LogDebug($"Testhost didn't exit gracefully '{testHostProcessInformation.ExitCode}')"); @@ -240,15 +378,91 @@ public async Task OnTestHostProcessExitedAsync(ITestHostProcessInformation testH [UnsupportedOSPlatform("ios")] [UnsupportedOSPlatform("tvos")] [UnsupportedOSPlatform("wasi")] - private async Task TakeDumpOfTreeAsync(CancellationToken cancellationToken) + private void TriggerDumpOnce(CancellationToken cancellationToken, bool triggeredByDeadline) { + // The inactivity timer and the deadline timer can both fire, and disposal can run + // concurrently. Claim the gate and publish the running dump task under the same lock, so both + // disposal paths (which take the lock, claim the gate, and capture _activityIndicatorTask) + // always observe and await the winning dump instead of tearing down the pipes underneath it. + CancellationTokenSource? handshakeCancellation; + lock (_dumpLock) + { + if (_dumpTaken != 0) + { + return; + } + + _dumpTaken = 1; + _activityIndicatorTask = TakeDumpOfTreeAsync(cancellationToken, triggeredByDeadline); + handshakeCancellation = _handshakeCancellationTokenSource; + } + + // Interrupt the pipe handshake, if one is still in flight. We are about to dump and kill the test + // host, and a host that wedged before connecting leaves that handshake waiting for a connection + // that will never arrive; killing it does not complete our own wait, so nothing else would end it + // before DefaultHangTimeSpanTimeout and the dump would never reach OnTestHostProcessExitedAsync + // to be published. Cancelled outside _dumpLock on purpose: the waiters' continuations can run + // inline here, and they continue into a handshake that takes _dumpLock again on its way out. + try + { + handshakeCancellation?.Cancel(); + } + catch (ObjectDisposedException) + { + // The handshake finished and disposed the source between the read above and this call, so + // there is nothing left to interrupt. + } + } + + private async Task TakeDumpOfTreeAsync(CancellationToken cancellationToken, bool triggeredByDeadline) + { + // This method is started synchronously inside the _dumpLock (see TriggerDumpOnce), which also + // publishes the returned task into _activityIndicatorTask. Yield immediately so none of the + // dump work runs while the lock is held: control returns to the caller, the task field is + // observed, and the lock is released before the (potentially slow) dump proceeds. HangDump runs + // out-of-process on a full runtime, so yielding to the thread pool here is safe. + await Task.Yield(); + ApplicationStateGuard.Ensure(_testHostProcessInformation is not null); - await _logger.LogInformationAsync($"Hang dump timeout({_activityTimerValue}) expired.").ConfigureAwait(false); - await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData(string.Format(CultureInfo.InvariantCulture, ExtensionResources.HangDumpTimeoutExpired, _activityTimerValue)), cancellationToken).ConfigureAwait(false); + string dumpReason = triggeredByDeadline + ? $"CI deadline approaching (dump scheduled at {_deadlineDumpAt:o})" + : $"Hang dump timeout({_activityTimerValue}) expired"; + + // Announcing the dump is diagnostics only, and it runs before the try/finally that kills the + // process tree. Loggers and output devices propagate exceptions, so letting one escape here would + // fault the dump task and leave the wedged host alive with no dump at all -- the exact situation + // this handler exists to resolve. Report the failure and take the dump anyway. + try + { + await _logger.LogInformationAsync($"{dumpReason}. Taking hang dump.").ConfigureAwait(false); + await _outputDisplay.DisplayAsync( + new ErrorMessageOutputDeviceData(triggeredByDeadline + ? ExtensionResources.HangDumpDeadlineApproaching + : string.Format(CultureInfo.InvariantCulture, ExtensionResources.HangDumpTimeoutExpired, _activityTimerValue)), + cancellationToken).ConfigureAwait(false); + } + catch (Exception e) + { + await LogBestEffortAsync("Could not announce the hang dump. Continuing with the dump.", e).ConfigureAwait(false); + } using IProcess process = _processHandler.GetProcessById(_testHostProcessInformation.PID); - var processTree = (await process.GetProcessTreeAsync(_logger, _outputDisplay, cancellationToken).ConfigureAwait(false)).Where(p => p.Process?.Name is not null and not "conhost" and not "WerFault").ToList(); + + // Walking the tree writes diagnostics through the same logger and output device, so it can fail + // for the same reason. Fall back to the root test host process: dumping and killing at least that + // one is what unblocks the run. + List processTree; + try + { + processTree = (await process.GetProcessTreeAsync(_logger, _outputDisplay, cancellationToken).ConfigureAwait(false)).Where(p => p.Process?.Name is not null and not "conhost" and not "WerFault").ToList(); + } + catch (Exception e) + { + await LogBestEffortAsync("Could not enumerate the test host process tree. Falling back to the root test host process.", e).ConfigureAwait(false); + processTree = [new ProcessTreeNode { Process = process, Level = 0 }]; + } + IEnumerable bottomUpTree = processTree.OrderByDescending(t => t.Level).Select(t => t.Process).OfType(); try @@ -267,27 +481,24 @@ private async Task TakeDumpOfTreeAsync(CancellationToken cancellationToken) await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData(string.Format(CultureInfo.InvariantCulture, ExtensionResources.DumpingProcess, process.Id, process.Name)), cancellationToken).ConfigureAwait(false); } - await _logger.LogInformationAsync($"Hang dump timeout({_activityTimerValue}) expired.").ConfigureAwait(false); + await _logger.LogInformationAsync($"{dumpReason}.").ConfigureAwait(false); - // Do not suspend processes with NetClient dumper it stops the diagnostic thread running in - // them and hang dump request will get stuck forever, because the process is not co-operating. - // Instead we start one task per dump asynchronously, and hope that the parent process will start dumping - // before the child process is done dumping. This way if the parent is waiting for the children to exit, - // we will be dumping it before it observes the child exiting and we get a more accurate results. If we did not - // do this, then parent that is awaiting child might exit before we get to dumping it. - foreach (IProcess p in bottomUpTree) - { - try + await QueryOnceAndDumpTreeAsync( + bottomUpTree, + GetInProgressTestsAsync, + async (p, inProgressTests, ct) => { - await TakeDumpAsync(p, cancellationToken).ConfigureAwait(false); - } - catch (Exception e) - { - // exceptions.Add(new InvalidOperationException($"Error while taking dump of process {p.Name} {p.Id}", e)); - await _logger.LogErrorAsync($"Error while taking dump of process {p.Id} - {p.Name}", e).ConfigureAwait(false); - await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData(string.Format(CultureInfo.InvariantCulture, ExtensionResources.ErrorWhileDumpingProcess, p.Id, p.Name, e)), cancellationToken).ConfigureAwait(false); - } - } + try + { + await TakeDumpAsync(p, inProgressTests, ct).ConfigureAwait(false); + } + catch (Exception e) + { + await _logger.LogErrorAsync($"Error while taking dump of process {p.Id} - {p.Name}", e).ConfigureAwait(false); + await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData(string.Format(CultureInfo.InvariantCulture, ExtensionResources.ErrorWhileDumpingProcess, p.Id, p.Name, e)), ct).ConfigureAwait(false); + } + }, + cancellationToken).ConfigureAwait(false); } finally { @@ -321,7 +532,131 @@ private async Task TakeDumpOfTreeAsync(CancellationToken cancellationToken) } } - private async Task TakeDumpAsync(IProcess process, CancellationToken cancellationToken) + /// + /// Reports a failure from a step that must never stop a dump from being taken or a wedged process + /// tree from being killed. Logging is itself best-effort here, because the failure being reported is + /// typically a logger or output device that is already throwing. + /// + private async Task LogBestEffortAsync(string message, Exception exception) + { + try + { + await _logger.LogErrorAsync(message, exception).ConfigureAwait(false); + } + catch (Exception) + { + // The logger is the thing that failed, so there is nothing left to report to. Swallowing is + // the whole point: the caller must continue to the dump and the process tree kill. + } + } + + /// + /// Asks once and then dumps every process in + /// , annotating each dump with that single answer. + /// + /// + /// The in-progress-test list describes the test host, so it is the same for every process in the tree, + /// while the query is bounded by . Asking per process would + /// multiply that bound by the size of the tree, and with a wedged consumer pipe a six-process tree would + /// spend the entire default 30s dump margin waiting before a single dump is written. + /// This is a separate method so that guarantee can be exercised with a fake tree and a stalled query: + /// itself dumps and then kills every process it walks, so a test + /// cannot drive it against a real process tree. + /// + internal static async Task QueryOnceAndDumpTreeAsync( + IEnumerable bottomUpTree, + Func> queryInProgressTestsAsync, + Func dumpProcessAsync, + CancellationToken cancellationToken) + { + (string, int)[] inProgressTests = await queryInProgressTestsAsync(cancellationToken).ConfigureAwait(false); + + // Do not suspend processes with NetClient dumper it stops the diagnostic thread running in + // them and hang dump request will get stuck forever, because the process is not co-operating. + // Instead we start one task per dump asynchronously, and hope that the parent process will start dumping + // before the child process is done dumping. This way if the parent is waiting for the children to exit, + // we will be dumping it before it observes the child exiting and we get a more accurate results. If we did not + // do this, then parent that is awaiting child might exit before we get to dumping it. + foreach (IProcess p in bottomUpTree) + { + await dumpProcessAsync(p, inProgressTests, cancellationToken).ConfigureAwait(false); + } + } + + /// + /// Asks the test host which tests are still running, so the dump can be annotated with them. + /// + /// + /// Called once per dump operation, not once per process: the answer describes the test host and the query + /// is bounded by , so repeating it for every process in the tree + /// would multiply that wait by the tree size and eat the dump margin. + /// The consumer pipe is only usable once the test host connected back over it. A non-null client is not + /// enough: it is created when the host sends its pipe name but only connected later, so a deadline dump + /// firing in that window (or a host that wedged during startup) would hit an unconnected pipe. The list is + /// therefore best-effort -- a connected-but-wedged host never replies and the app token is not cancelled + /// mid-run, so any failure is logged and swallowed and an empty list is returned, and it can never block + /// taking the dump and killing the tree. + /// + private Task<(string, int)[]> GetInProgressTestsAsync(CancellationToken cancellationToken) + { + NamedPipeClient? namedPipeClient = _namedPipeClient; + return namedPipeClient is null + ? Task.FromResult<(string, int)[]>([]) + : QueryInProgressTestsWithTimeoutAsync( + async queryCancellationToken => + { + GetInProgressTestsResponse tests = await namedPipeClient.RequestReplyAsync(new GetInProgressTestsRequest(), queryCancellationToken).ConfigureAwait(false); + return tests.Tests; + }, + InProgressTestsQueryTimeout, + ex => _logger.LogDebugAsync($"Could not collect the in-progress tests before dumping (the consumer pipe may not be connected, or the host did not reply within {InProgressTestsQueryTimeout}). Continuing with the dump. {ex}"), + cancellationToken); + } + + /// + /// Runs under a bound of , and + /// returns an empty list if it does not answer in time or fails -- including when reporting that failure + /// itself fails. + /// + /// + /// The bound lives here rather than in the caller's delegate so it is the product, not the caller, that + /// gives up on a connected-but-wedged host: the application token is not cancelled while the run is still + /// "in progress", which is exactly when the deadline dump fires, so an unbounded request/reply would block + /// the dump and the kill indefinitely and consume the whole dump margin. + /// This is a separate method so that bound can be exercised with a reply that never arrives: the real + /// request/reply goes over a named pipe to another process, which a unit test cannot stand up. + /// + internal static async Task<(string, int)[]> QueryInProgressTestsWithTimeoutAsync( + Func> requestInProgressTestsAsync, + TimeSpan timeout, + Func logFailureAsync, + CancellationToken cancellationToken) + { + try + { + using var queryCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + queryCts.CancelAfter(timeout); + return await requestInProgressTestsAsync(queryCts.Token).ConfigureAwait(false); + } + catch (Exception ex) + { + // The empty-list fallback is the whole point of this method, so it must survive a failing + // diagnostic too. logFailureAsync is a logger call and logger providers can fail; letting that + // throw would escape the caller, which is explicitly best-effort, and skip the dump entirely. + try + { + await logFailureAsync(ex).ConfigureAwait(false); + } + catch (Exception) + { + // Ignore: taking the dump matters more than recording why the query could not answer. + } + + return []; + } + } + + private async Task TakeDumpAsync(IProcess process, (string, int)[] inProgressTests, CancellationToken cancellationToken) { ApplicationStateGuard.Ensure(_testHostProcessInformation is not null); ApplicationStateGuard.Ensure(_dumpType is not null); @@ -355,23 +690,31 @@ private async Task TakeDumpAsync(IProcess process, CancellationToken cancellatio // Ensure the destination directory exists (templates may include directory separators, e.g. {asm}/{pname}). Directory.CreateDirectory(Path.GetDirectoryName(finalDumpFileName)!); - ApplicationStateGuard.Ensure(_namedPipeClient is not null); - GetInProgressTestsResponse tests = await _namedPipeClient.RequestReplyAsync(new GetInProgressTestsRequest(), cancellationToken).ConfigureAwait(false); - if (tests.Tests.Length > 0) + // The in-progress tests were queried once for the whole dump operation (see GetInProgressTestsAsync); + // write them next to this dump so the dump can be read together with what was running. + if (inProgressTests.Length > 0) { - string hangTestsFileName = Path.ChangeExtension(finalDumpFileName, ".log"); - using (FileStream fs = File.OpenWrite(hangTestsFileName)) - using (StreamWriter sw = new(fs)) + try { - await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData(ExtensionResources.RunningTestsWhileDumping), cancellationToken).ConfigureAwait(false); - foreach ((string testName, int seconds) in tests.Tests) + string hangTestsFileName = Path.ChangeExtension(finalDumpFileName, ".log"); + using (FileStream fs = File.OpenWrite(hangTestsFileName)) + using (StreamWriter sw = new(fs)) { - await sw.WriteLineAsync($"[{TimeSpan.FromSeconds(seconds)}] {testName}").ConfigureAwait(false); - await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData($"[{TimeSpan.FromSeconds(seconds)}] {testName}"), cancellationToken).ConfigureAwait(false); + await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData(ExtensionResources.RunningTestsWhileDumping), cancellationToken).ConfigureAwait(false); + foreach ((string testName, int seconds) in inProgressTests) + { + await sw.WriteLineAsync($"[{TimeSpan.FromSeconds(seconds)}] {testName}").ConfigureAwait(false); + await _outputDisplay.DisplayAsync(new ErrorMessageOutputDeviceData($"[{TimeSpan.FromSeconds(seconds)}] {testName}"), cancellationToken).ConfigureAwait(false); + } } - } - await _messageBus.PublishAsync(this, new FileArtifact(new FileInfo(hangTestsFileName), ExtensionResources.HangTestListArtifactDisplayName, ExtensionResources.HangTestListArtifactDescription)).ConfigureAwait(false); + await _messageBus.PublishAsync(this, new FileArtifact(new FileInfo(hangTestsFileName), ExtensionResources.HangTestListArtifactDisplayName, ExtensionResources.HangTestListArtifactDescription)).ConfigureAwait(false); + } + catch (Exception ex) + { + // Writing the list is a convenience; it must never block taking the dump and killing the tree. + await _logger.LogDebugAsync($"Could not write the in-progress tests next to the dump of process {process.Id}. Continuing with the dump. {ex}").ConfigureAwait(false); + } } await _logger.LogInformationAsync($"Creating dump filename {finalDumpFileName}").ConfigureAwait(false); @@ -450,12 +793,28 @@ private static void NotifyCrashDumpServiceIfEnabled() public void Dispose() { - if (_activityIndicatorTask is not null) + // Stop the deadline and inactivity timers so no callback can start a new dump while we tear + // down the pipes. The happy path disposes them in OnTestHostProcessExitedAsync, but that runs + // only on a clean exit; Ctrl+C or an exception skips it, so dispose here too (Timer.Dispose is + // idempotent, so disposing twice is safe). + _deadlineTimer?.Dispose(); + _activityTimer?.Dispose(); + + Task? activityIndicatorTask; + lock (_dumpLock) + { + // Claim the gate so no timer callback can start a new dump once we begin tearing down the + // pipes, and capture any dump already in flight so we wait for it below. + _dumpTaken = 1; + activityIndicatorTask = _activityIndicatorTask; + } + + if (activityIndicatorTask is not null) { bool waitResult; try { - waitResult = _activityIndicatorTask.Wait(TimeoutHelper.DefaultHangTimeSpanTimeout); + waitResult = activityIndicatorTask.Wait(TimeoutHelper.DefaultHangTimeSpanTimeout); } catch (Exception e) { @@ -477,11 +836,27 @@ public void Dispose() #if NETCOREAPP public async ValueTask DisposeAsync() { - if (_activityIndicatorTask is not null) + // Stop the deadline and inactivity timers so no callback can start a new dump while we tear + // down the pipes. The happy path disposes them in OnTestHostProcessExitedAsync, but that runs + // only on a clean exit; Ctrl+C or an exception skips it, so dispose here too (Timer.Dispose is + // idempotent, so disposing twice is safe). + _deadlineTimer?.Dispose(); + _activityTimer?.Dispose(); + + Task? activityIndicatorTask; + lock (_dumpLock) + { + // Claim the gate so no timer callback can start a new dump once we begin tearing down the + // pipes, and capture any dump already in flight so we await it below. + _dumpTaken = 1; + activityIndicatorTask = _activityIndicatorTask; + } + + if (activityIndicatorTask is not null) { try { - await _activityIndicatorTask.TimeoutAfterAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false); + await activityIndicatorTask.TimeoutAfterAsync(TimeoutHelper.DefaultHangTimeSpanTimeout).ConfigureAwait(false); } catch (Exception e) { diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.HangDump/InternalAPI/InternalAPI.Unshipped.txt index bc570abc18..19e2e459d5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/InternalAPI/InternalAPI.Unshipped.txt @@ -10,6 +10,7 @@ static Microsoft.Testing.Platform.IPC.NamedPipeServer.ResolvePipeDirectory(strin static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Platform.Services.ArtifactNamingHelper.ResolveAndSanitize(string! template, string! processName, string! processId, System.DateTimeOffset timestamp, System.Func! sanitizeLeafFileName) -> string! static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.ReadFields(System.IO.Stream! stream, System.Func! tryReadField) -> void static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.TryReadExecutionScopedField(System.IO.Stream! stream, ushort fieldId, int fieldSize, ref string? executionId, ref string? instanceId) -> bool @@ -23,3 +24,13 @@ Microsoft.Testing.Platform.IPC.NamedPipeConnectionBase.WriteMessageAsync(System. *REMOVED*Microsoft.Testing.Platform.IPC.NamedPipeConnectionBase.WriteMessageAsync(System.IO.Pipes.PipeStream! stream, Microsoft.Testing.Platform.IPC.INamedPipeSerializer! serializer, object! message, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! Microsoft.Testing.Platform.IPC.NamedPipeServer.NamedPipeServer(Microsoft.Testing.Platform.IPC.PipeNameDescription! pipeNameDescription, System.Func!>! callback, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Logging.ILogger! logger, Microsoft.Testing.Platform.Helpers.ITask! task, int maxNumberOfServerInstances, System.Collections.Generic.IReadOnlyList? authorizedSecurityIdentities, System.Threading.CancellationToken cancellationToken) -> void Microsoft.Testing.Platform.IPC.NamedPipeServer.NamedPipeServer(string! name, System.Func!>! callback, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Logging.ILogger! logger, Microsoft.Testing.Platform.Helpers.ITask! task, System.Collections.Generic.IReadOnlyList? authorizedSecurityIdentities, System.Threading.CancellationToken cancellationToken) -> void +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string! +Microsoft.Testing.Platform.Helpers.DeadlineHelper +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.TryGetDeadline(Microsoft.Testing.Platform.Helpers.IEnvironment! environment, out System.DateTimeOffset deadlineUtc) -> bool +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.GetStopMargin(Microsoft.Testing.Platform.Helpers.IEnvironment! environment) -> System.TimeSpan +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.GetDumpMargin(Microsoft.Testing.Platform.Helpers.IEnvironment! environment) -> System.TimeSpan +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.SubtractSaturating(System.DateTimeOffset instant, System.TimeSpan margin) -> System.DateTimeOffset +static Microsoft.Testing.Extensions.Diagnostics.HangDumpProcessLifetimeHandler.QueryOnceAndDumpTreeAsync(System.Collections.Generic.IEnumerable! bottomUpTree, System.Func!>! queryInProgressTestsAsync, System.Func! dumpProcessAsync, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +static Microsoft.Testing.Extensions.Diagnostics.HangDumpProcessLifetimeHandler.QueryInProgressTestsWithTimeoutAsync(System.Func!>! requestInProgressTestsAsync, System.TimeSpan timeout, System.Func! logFailureAsync, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task<(string!, int)[]!>! diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Microsoft.Testing.Extensions.HangDump.csproj b/src/Platform/Microsoft.Testing.Extensions.HangDump/Microsoft.Testing.Extensions.HangDump.csproj index fed93f4f2c..b486d73161 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Microsoft.Testing.Extensions.HangDump.csproj +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Microsoft.Testing.Extensions.HangDump.csproj @@ -40,6 +40,7 @@ $(CommonProductDescription)]]> + diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/ExtensionResources.resx b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/ExtensionResources.resx index 18fe4d6cfd..b5613801c5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/ExtensionResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/ExtensionResources.resx @@ -146,6 +146,9 @@ Hang dump file + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + Environment variable '{0}' is set to '{1}' instead of '{2}' {0} is the environment variable name. {1} is the actual value. {2} is the expected value. diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.cs.xlf index d5d9de5c7c..2d83cc86a5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.cs.xlf @@ -42,6 +42,11 @@ Soubor výpisu paměti při zablokování + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' Proměnná prostředí {0} je nastavená na {1} místo na {2}. diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.de.xlf index 038fb8d15a..1b3f99f129 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.de.xlf @@ -42,6 +42,11 @@ Absturzspeicherabbilddatei + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' Umgebungsvariable "{0}" ist auf "{1}" anstatt auf "{2}" festgelegt. diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.es.xlf index 4155fad207..27d863ae03 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.es.xlf @@ -42,6 +42,11 @@ Archivo de volcado de bloqueo + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' La variable de entorno '{0}' se establece en '{1}' en lugar de '{2}' diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.fr.xlf index e4dba49858..b090a365cb 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.fr.xlf @@ -42,6 +42,11 @@ Bloquer le fichier de vidage + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' La variable d’environnement «{0}» a la valeur «{1}» au lieu de «{2}» diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.it.xlf index f53d10c554..15ee698ce9 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.it.xlf @@ -42,6 +42,11 @@ File dump di blocco + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' La variabile di ambiente '{0}' è impostata su '{1}' anziché su '{2}' diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ja.xlf index d68c9c0698..d6501af964 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ja.xlf @@ -42,6 +42,11 @@ ハング ダンプ ファイル + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' 環境変数 '{0}' は '{2}' ではなく '{1}' に設定されています diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ko.xlf index 4d0ec3f9d3..4544a4617b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ko.xlf @@ -42,6 +42,11 @@ 중단 덤프 파일 + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' 환경 변수 '{0}'이(가) '{2}' 대신 '{1}'로 설정되어 있습니다. diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pl.xlf index 9d378503a3..fa0a0d9c11 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pl.xlf @@ -42,6 +42,11 @@ Zawieszanie pliku zrzutu + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' Zmienna środowiskowa „{0}” jest ustawiona na wartość „{1}” zamiast „{2}” diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pt-BR.xlf index 97ff37cdc5..812584f522 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.pt-BR.xlf @@ -42,6 +42,11 @@ Arquivo de despejo de travamento + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' A variável de ambiente ''{0}'' ambiente está definida como ''{1}'' em vez de ''{2}'' diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ru.xlf index a495ff60a5..dddfb6d312 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.ru.xlf @@ -42,6 +42,11 @@ Файл дампа зависания + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' Переменной среды "{0}" присвоено значение "{1}" вместо "{2}" diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.tr.xlf index 18d72656ed..2595f9ac1d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.tr.xlf @@ -42,6 +42,11 @@ Döküm dosyasını as + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' '{0}' ortam değişkeni '{1}' yerine '{2}' olarak ayarlandı diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hans.xlf index e2b6f1f908..b866fc3e0e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hans.xlf @@ -42,6 +42,11 @@ 挂起转储文件 + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' 环境变量“{0}”设置为“{1}”，而不是“{2}” diff --git a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hant.xlf index 12b9af2296..2c8eec9155 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.HangDump/Resources/xlf/ExtensionResources.zh-Hant.xlf @@ -42,6 +42,11 @@ 擱置傾印檔案 + + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + CI deadline approaching: taking a hang dump before the run is hard-cancelled. + + Environment variable '{0}' is set to '{1}' instead of '{2}' 環境變數 '{0}' 已設定為 '{1}' 而不是 '{2}' diff --git a/src/Platform/Microsoft.Testing.Extensions.HotReload/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.HotReload/InternalAPI/InternalAPI.Unshipped.txt index 53cce0d029..deac51a1e2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HotReload/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.HotReload/InternalAPI/InternalAPI.Unshipped.txt @@ -1,3 +1,6 @@ #nullable enable const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_ATTEMPTNUMBER = "TESTINGPLATFORM_DOTNETTEST_ATTEMPTNUMBER" -> string! const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_PIPE_DIRECTORY = "TESTINGPLATFORM_PIPE_DIRECTORY" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/InternalAPI/InternalAPI.Unshipped.txt index c2e7bd3f92..cbcd78d835 100644 --- a/src/Platform/Microsoft.Testing.Extensions.HtmlReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.HtmlReport/InternalAPI/InternalAPI.Unshipped.txt @@ -26,4 +26,5 @@ virtual Microsoft.Testing.Extensions.ReportGeneratorBase string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Platform.Services.ArtifactNamingHelper.ResolveAndSanitize(string! template, string! processName, string! processId, System.DateTimeOffset timestamp, System.Func! sanitizeLeafFileName) -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.JUnitReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.JUnitReport/InternalAPI/InternalAPI.Unshipped.txt index ccd1e6ef90..cae2bf4ac7 100644 --- a/src/Platform/Microsoft.Testing.Extensions.JUnitReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.JUnitReport/InternalAPI/InternalAPI.Unshipped.txt @@ -25,6 +25,7 @@ static Microsoft.Testing.Extensions.JUnitReport.JUnitReportMerger.MergeToFileAsy static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Platform.Services.ArtifactNamingHelper.ResolveAndSanitize(string! template, string! processName, string! processId, System.DateTimeOffset timestamp, System.Func! sanitizeLeafFileName) -> string! Microsoft.Testing.Extensions.MergeOutputFileHelper static Microsoft.Testing.Extensions.MergeOutputFileHelper.BuildCaseFoldedProbePath(string! directory, string! probeFileName) -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.MSBuild/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.MSBuild/InternalAPI.Unshipped.txt index aba9805522..76a4147d2d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.MSBuild/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.MSBuild/InternalAPI.Unshipped.txt @@ -10,6 +10,7 @@ static Microsoft.Testing.Platform.IPC.NamedPipeServer.ResolvePipeDirectory(strin static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.ReadFields(System.IO.Stream! stream, System.Func! tryReadField) -> void static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.TryReadExecutionScopedField(System.IO.Stream! stream, ushort fieldId, int fieldSize, ref string? executionId, ref string? instanceId) -> bool static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.WriteExecutionScopedHeader(System.IO.Stream! stream, string? executionId, string? instanceId, ushort payloadFieldCount) -> void @@ -22,3 +23,6 @@ Microsoft.Testing.Platform.IPC.NamedPipeConnectionBase.WriteMessageAsync(System. *REMOVED*Microsoft.Testing.Platform.IPC.NamedPipeConnectionBase.WriteMessageAsync(System.IO.Pipes.PipeStream! stream, Microsoft.Testing.Platform.IPC.INamedPipeSerializer! serializer, object! message, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! Microsoft.Testing.Platform.IPC.NamedPipeServer.NamedPipeServer(Microsoft.Testing.Platform.IPC.PipeNameDescription! pipeNameDescription, System.Func!>! callback, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Logging.ILogger! logger, Microsoft.Testing.Platform.Helpers.ITask! task, int maxNumberOfServerInstances, System.Collections.Generic.IReadOnlyList? authorizedSecurityIdentities, System.Threading.CancellationToken cancellationToken) -> void Microsoft.Testing.Platform.IPC.NamedPipeServer.NamedPipeServer(string! name, System.Func!>! callback, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Logging.ILogger! logger, Microsoft.Testing.Platform.Helpers.ITask! task, System.Collections.Generic.IReadOnlyList? authorizedSecurityIdentities, System.Threading.CancellationToken cancellationToken) -> void +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.Retry/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.Retry/InternalAPI/InternalAPI.Unshipped.txt index 8b7e084b1b..9af13931bd 100644 --- a/src/Platform/Microsoft.Testing.Extensions.Retry/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.Retry/InternalAPI/InternalAPI.Unshipped.txt @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DOTNETTEST_ATTEMPTNUMBER = "TESTINGPLATFORM_DOTNETTEST_ATTEMPTNUMBER" -> string! const Microsoft.Testing.Platform.IPC.FileArtifactMessageFieldsId.InputArtifactPaths = 8 -> ushort const Microsoft.Testing.Platform.IPC.FileArtifactMessageFieldsId.Kind = 7 -> ushort @@ -14,6 +14,7 @@ static Microsoft.Testing.Platform.IPC.NamedPipeServer.ResolvePipeDirectory(strin static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.ReadFields(System.IO.Stream! stream, System.Func! tryReadField) -> void static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.TryReadExecutionScopedField(System.IO.Stream! stream, ushort fieldId, int fieldSize, ref string? executionId, ref string? instanceId) -> bool static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.WriteExecutionScopedHeader(System.IO.Stream! stream, string? executionId, string? instanceId, ushort payloadFieldCount) -> void @@ -88,3 +89,6 @@ Microsoft.Testing.Extensions.Policy.RetryAttemptArtifact.DestinationPath.get -> Microsoft.Testing.Extensions.Policy.RetryAttemptArtifact.Kind.get -> string? Microsoft.Testing.Extensions.Policy.RetryAttemptArtifact.Path.get -> string! static Microsoft.Testing.Extensions.Policy.RetrySummaryReporter.MoveArtifactsAsync(Microsoft.Testing.Platform.Extensions.OutputDevice.IOutputDeviceDataProducer! producer, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Helpers.IFileSystem! fileSystem, Microsoft.Testing.Platform.Logging.ILogger! logger, string! currentTryResultFolder, string! resultDirectory, System.Collections.Generic.IReadOnlyDictionary! replacements, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string! diff --git a/src/Platform/Microsoft.Testing.Extensions.TrxReport/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Extensions.TrxReport/InternalAPI/InternalAPI.Unshipped.txt index 276d0b4ab9..b1378bd0d2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.TrxReport/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Extensions.TrxReport/InternalAPI/InternalAPI.Unshipped.txt @@ -63,6 +63,7 @@ static Microsoft.Testing.Platform.IPC.NamedPipeServer.ResolvePipeDirectory(strin static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipeDirectoryNotWritableErrorMessage.get -> string! static Microsoft.Testing.Platform.Resources.PlatformResources.NamedPipePathTooLongErrorMessage.get -> string! Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode static Microsoft.Testing.Platform.Services.ArtifactNamingHelper.ResolveAndSanitize(string! template, string! processName, string! processId, System.DateTimeOffset timestamp, System.Func! sanitizeLeafFileName) -> string! static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.ReadFields(System.IO.Stream! stream, System.Func! tryReadField) -> void static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.TryReadExecutionScopedField(System.IO.Stream! stream, ushort fieldId, int fieldSize, ref string? executionId, ref string? instanceId) -> bool @@ -70,6 +71,9 @@ static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.WriteExecutionS static Microsoft.Testing.Platform.IPC.Serializers.BaseSerializer.WriteListPayload(System.IO.Stream! stream, ushort fieldId, T[]? list, System.Action! writeItem) -> void const Microsoft.Testing.Platform.IPC.FailedTestResultMessageFieldsId.Expected = 10 -> ushort const Microsoft.Testing.Platform.IPC.FailedTestResultMessageFieldsId.Actual = 11 -> ushort +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string! Microsoft.Testing.Extensions.MergeOutputFileHelper static Microsoft.Testing.Extensions.MergeOutputFileHelper.BuildCaseFoldedProbePath(string! directory, string! probeFileName) -> string! static Microsoft.Testing.Extensions.MergeOutputFileHelper.EnsureOutputDoesNotAliasInput(System.Collections.Generic.IReadOnlyList! inputPaths, string! outputPath) -> void diff --git a/src/Platform/Microsoft.Testing.Platform/Extensions/AbortAtDeadlineExtension.cs b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortAtDeadlineExtension.cs new file mode 100644 index 0000000000..768b07ca6a --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Extensions/AbortAtDeadlineExtension.cs @@ -0,0 +1,551 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Resources; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Platform.Extensions; + +/// +/// Reacts to a CI-imposed hard-cancel deadline (see ): a short margin +/// before the deadline it asks the test framework to gracefully stop scheduling new tests, so the +/// in-flight session can end normally and all reporters (TRX/HTML/AzDO live run) get to finalize +/// before the CI runner hard-kills the process. +/// +/// +/// This is a prototype. It is timer-driven (the deadline is an absolute instant, so the timer is +/// armed at construction). It implements so the message bus keeps a live +/// reference to it for the duration of the run (which also keeps its timer alive); it consumes no +/// message types, so the bus keeps that reference without routing any message to it. It also implements +/// as a backstop and is registered as a service so the host can call +/// the moment the test framework invoker returns. That early signal +/// prevents a timer firing while reporters finalize an already-finished run from marking it deadline-truncated. +/// +internal sealed class AbortAtDeadlineExtension : IDataConsumer, ITestSessionLifetimeHandler, IOutputDeviceDataProducer, IDisposable +#if NETCOREAPP +#pragma warning disable SA1001 // Commas should be spaced correctly + , IAsyncDisposable +#pragma warning restore SA1001 // Commas should be spaced correctly +#endif +{ + private readonly IGracefulStopTestExecutionCapability? _capability; + private readonly IStopPoliciesService _policiesService; + private readonly ITestApplicationCancellationTokenSource _cancellationTokenSource; + private readonly IOutputDevice _outputDevice; + private readonly ILogger _logger; + private readonly DateTimeOffset? _stopAt; + private readonly Timer? _timer; + + // How long a single best-effort diagnostic may take before it is abandoned. Injectable only so a test + // can exercise the bound without waiting DefaultReportTimeout for it; production always uses the default. + private readonly TimeSpan _reportTimeout; + + // Serializes publishing _handleDeadlineTask against Dispose reading it, so the timer callback and + // disposal cannot interleave in a way that starts the handler after Dispose has already returned. +#if NET9_0_OR_GREATER + private readonly Lock _lock = new(); +#else + private readonly object _lock = new(); +#endif + private int _handled; + private volatile bool _disposed; + + // Which of "test execution finished" and "the deadline took the run" happened first. Both transitions are + // made under _lock and only out of Running, so they are mutually exclusive: whichever takes the lock first + // wins and the other becomes a no-op. Read without the lock on the timer callback's fast-path, so it is + // volatile. + private volatile RunState _state; + private Task? _handleDeadlineTask; + + /// + /// Bounded wait applied on disposal to let an in-flight deadline handler finish reporting before + /// the host tears down, without letting a wedged stop hang disposal forever. + /// + private static readonly TimeSpan DisposeDrainTimeout = TimeSpan.FromSeconds(30); + + /// + /// Bounded wait applied to each best-effort diagnostic on the deadline path, so a logger or output + /// device that never completes cannot hold up the graceful stop it precedes. + /// + /// + /// Generous enough that a healthy provider never hits it, and short relative to the margin the + /// deadline leaves for the stop to take effect. + /// + private static readonly TimeSpan DefaultReportTimeout = TimeSpan.FromSeconds(10); + + /// + /// throws for due times above ~49.7 days (its internal limit is + /// milliseconds). A deadline that far out is effectively "never" + /// for a test run, so we clamp to this maximum instead of throwing at construction. + /// + private static readonly TimeSpan MaxTimerDueTime = TimeSpan.FromMilliseconds(uint.MaxValue - 1); + + public AbortAtDeadlineExtension( + IEnvironment environment, + IClock clock, + IGracefulStopTestExecutionCapability? capability, + IStopPoliciesService policiesService, + ITestApplicationCancellationTokenSource cancellationTokenSource, + IOutputDevice outputDevice, + ILoggerFactory loggerFactory, + TimeSpan? reportTimeout = null) + { + _capability = capability; + _policiesService = policiesService; + _cancellationTokenSource = cancellationTokenSource; + _outputDevice = outputDevice; + _logger = loggerFactory.CreateLogger(nameof(AbortAtDeadlineExtension)); + _reportTimeout = reportTimeout ?? DefaultReportTimeout; + + if (!DeadlineHelper.TryGetDeadline(environment, out DateTimeOffset deadline)) + { + // Distinguish "opt-in is off" (variable unset) from "set but malformed". The former is the + // normal case and stays silent; the latter is a configuration mistake worth a warning. + string? raw = environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE); + if (!RoslynString.IsNullOrWhiteSpace(raw)) + { + TryLog(() => _logger.LogWarning($"Environment variable '{EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE}' is set to '{raw}' but could not be parsed as an absolute ISO 8601 instant. Deadline-aware cancellation is disabled.")); + } + + return; + } + + TimeSpan stopMargin = DeadlineHelper.GetStopMargin(environment); + TimeSpan dumpMargin = DeadlineHelper.GetDumpMargin(environment); + DateTimeOffset stopAt = DeadlineHelper.SubtractSaturating(deadline, stopMargin); + + // A deadline given without an offset is parsed as UTC (AssumeUniversal), which is easy to get + // wrong. Log the resolved instants and margins so a misconfigured offset is visible. + TryLog(() => + { + _logger.LogInformation($"Deadline-aware cancellation: deadline={deadline:o}, stopMargin={stopMargin}, dumpMargin={dumpMargin}, graceful stop scheduled at {stopAt:o} (UTC)."); + + // stopMargin is meant to be larger than dumpMargin so the graceful stop is attempted before + // the hang dump. Warn when the ordering is inverted rather than silently misbehaving. + if (dumpMargin >= stopMargin) + { + _logger.LogWarning($"Deadline dump margin ({dumpMargin}) is greater than or equal to the stop margin ({stopMargin}). The graceful stop is meant to run before the hang dump; with these margins the hang dump may fire first."); + } + }); + + if (capability is null) + { + // A deadline is configured but this framework cannot stop gracefully, so nothing is armed. + // Surface it rather than silently doing nothing. + TryLog(() => _logger.LogWarning($"Environment variable '{EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE}' is set but the test framework does not support graceful stop ('{nameof(IGracefulStopTestExecutionCapability)}'); the platform cannot stop early at the deadline.")); + return; + } + + _stopAt = stopAt; + + // The deadline is absolute wall-clock time, so we can arm a one-shot timer now. If the + // computed instant is already in the past, fire as soon as possible. If it is farther + // out than a Timer can represent, clamp it: the run (and this timer) will be disposed + // long before the clamped due time elapses, so the timer never fires early in practice. + TimeSpan dueTime = stopAt - clock.UtcNow; + if (dueTime < TimeSpan.Zero) + { + dueTime = TimeSpan.Zero; + } + else if (dueTime > MaxTimerDueTime) + { + dueTime = MaxTimerDueTime; + } + + _timer = new Timer(static state => ((AbortAtDeadlineExtension)state!).OnDeadlineReached(), this, dueTime, Timeout.InfiniteTimeSpan); + } + + private static void TryLog(Action logAction) + { + try + { + logAction(); + } + catch (Exception) + { + // Construction-time diagnostics are best-effort: a logger failure must never break test + // framework construction. + } + } + + // No message types are consumed: this extension implements IDataConsumer only to keep a live + // reference on the message bus (see the remark on the class). Returning an empty list avoids the + // bus routing every test result to a no-op ConsumeAsync, which would be O(test-count) overhead. + public Type[] DataTypesConsumed { get; } = []; + + /// + public string Uid => nameof(AbortAtDeadlineExtension); + + /// + public string Version => PlatformVersion.Version; + + /// + public string DisplayName => nameof(AbortAtDeadlineExtension); + + /// + public string Description { get; } = PlatformResources.AbortAtDeadlineDescription; + + /// + public Task IsEnabledAsync() => Task.FromResult(_stopAt.HasValue && _capability is not null); + + /// + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + => Task.CompletedTask; + + /// + public Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + => Task.CompletedTask; + + /// + public Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + NotifyTestExecutionCompleted(); + return Task.CompletedTask; + } + + /// + /// Disarms the deadline because test execution has finished. + /// + /// + /// The host calls this as soon as the test framework invoker returns, before end-of-session draining and + /// reporting begin. That instant matters: from here on the deadline is moot, because every test that was + /// going to run has run. Doing it through instead would be far + /// too late -- session-end notification first drains the message bus, then runs the non-consumer handlers, + /// then the consumer handlers in registration order, and this extension is a consumer appended after the + /// reporters. A deadline reached anywhere in that window would still mark a fully-executed run as + /// truncated (exit code 15). + /// The transition is made under the same lock uses, and only out of + /// , so it is atomic against the deadline handler committing its verdict: + /// either this call wins and the handler then abandons the stop, or the handler claimed the run first and + /// this call leaves that verdict alone (a real truncation must not be un-marked by the completion that the + /// requested stop itself produced). + /// The timer itself is left to be disposed at host teardown; the state is what makes any late fire a no-op. + /// + public void NotifyTestExecutionCompleted() + { + lock (_lock) + { + if (_state == RunState.Running) + { + _state = RunState.Completed; + } + } + } + + /// + /// Takes the run for the deadline, if test execution has not finished (or disposal started) first. + /// + /// + /// This is the commit point of the deadline verdict, and it is deliberately the last thing before the + /// verdict is recorded: the handler yields and reports before getting here, and the test framework invoker + /// can return during that window. Checking completion only before starting the handler would therefore + /// still let a run that executed every test be reported as deadline-truncated. + /// + /// when the deadline still applies and this call owns the verdict. + private bool TryClaimDeadline() + { + lock (_lock) + { + if (_disposed || _state != RunState.Running || _policiesService.IsTestExecutionCompleted) + { + return false; + } + + _state = RunState.DeadlineClaimed; + return true; + } + } + + /// + /// Gives the run back after a claimed deadline turned out not to stop it, so a completion arriving later + /// is recorded normally. + /// + private void ReleaseDeadlineClaim() + { + lock (_lock) + { + if (_state == RunState.DeadlineClaimed) + { + _state = RunState.Running; + } + } + } + + private void OnDeadlineReached() + { + // Do not start deadline handling once we are tearing down, or once test execution has already + // completed (cheap fast-path; both are re-checked under the lock below to actually close the race + // with Dispose and NotifyTestExecutionCompleted). + if (_disposed || _state != RunState.Running || _policiesService.IsTestExecutionCompleted) + { + return; + } + + // Ensure we react only once. + if (Interlocked.Exchange(ref _handled, 1) != 0) + { + return; + } + + // Publish the handler task under the same lock Dispose uses so the two cannot interleave: + // either we set _handleDeadlineTask before Dispose captures it (Dispose then drains it), or + // Dispose sets _disposed first and we observe it here and never start the handler against + // torn-down services. Without this, the timer callback could pass the _disposed check above, + // Dispose could run to completion (seeing _handleDeadlineTask still null, so draining + // nothing), and only then would the handler start -- after disposal already returned. + lock (_lock) + { + // Bail if disposal started, or if test execution finished between the fast-path check above and + // acquiring the lock (the timer fired while the reporters are finalizing a fully-completed run). In + // either case there is nothing left to stop, and marking the run deadline-truncated would wrongly + // force exit code 15 on a run that actually completed. This is only an early-out: the handler + // re-checks with TryClaimDeadline right before committing the verdict, which is what actually + // closes the race. + if (_disposed || _state != RunState.Running || _policiesService.IsTestExecutionCompleted) + { + return; + } + + // Start the handler and capture its task while holding the lock so it cannot interleave with + // Dispose. HandleDeadlineAsync yields immediately (await Task.Yield()), so none of its work runs + // while the lock is held: it returns an incomplete task here, we store it, and the lock is + // released before the handler body executes. That keeps the lock's scope to just publishing the + // task and avoids running the graceful-stop callback synchronously under the lock -- which could + // deadlock if the stop waits on session finishing/disposal (both of which take this lock). + _handleDeadlineTask = HandleDeadlineAsync(); + } + } + + private async Task HandleDeadlineAsync() + { + // This method is started synchronously inside _lock (see OnDeadlineReached), which also publishes + // the returned task into _handleDeadlineTask. An async method does NOT necessarily yield at its + // first await: the logger, output device, the empty deadline-callback queue and the framework's + // graceful-stop capability can all return already-completed tasks, which would run this whole + // handler synchronously while _lock is held. If the graceful stop then synchronously waited on + // session finishing or disposal (both take _lock), that would deadlock. Yield first so control + // returns to the caller, _handleDeadlineTask is observed, and _lock is released before any handler + // work runs. Task.Yield posts the continuation back to the current context, so it is cooperative + // and safe even on a single-threaded runtime (browser/WASI) -- unlike Task.Run, which needs another + // thread to pick the work up. + await Task.Yield(); + + if (_capability is not { } capability) + { + return; + } + + // Diagnostics are best-effort and must never prevent the graceful stop below: MTP's logger + // and output device both propagate provider exceptions, so a failure here would otherwise + // skip the deadline handling entirely. + await TryReportAsync( + () => _logger.LogInformationAsync($"Deadline approaching (stop scheduled at {_stopAt:o}). Requesting graceful stop of test execution."), + "Failed to report the approaching deadline.").ConfigureAwait(false); + + // Take the run for the deadline, atomically against NotifyTestExecutionCompleted. Everything above -- + // the yield that got us off the timer callback, and the logging -- runs while the test framework + // invoker may still return, so this is the first point at which committing the verdict is safe. + if (!TryClaimDeadline()) + { + // Test execution finished (or disposal started) while we were getting here. There is nothing left + // to stop and the run was not truncated, so neither the verdict nor the stop request may happen: + // both would report a deadline stop for a run that executed every test. + await TryReportAsync( + () => _logger.LogDebugAsync("Test execution completed while the approaching deadline was being reported; abandoning the graceful stop."), + "Failed to report the abandoned deadline stop.").ConfigureAwait(false); + return; + } + + bool stopAccepted = false; + try + { + // Commit the verdict with nothing awaited between it and the claim above. The claim closes the + // door on NotifyTestExecutionCompleted -- once the run is claimed, a completion can no longer + // disarm the deadline -- so any await in between would be a window where the run finishes on its + // own and we still go on to report it as truncated. Setting the flag is synchronous inside + // ExecuteDeadlineCallbacksAsync, so claim and commit are effectively one step. + // + // It also has to happen BEFORE the stop is requested. StopTestExecutionAsync can unblock the + // framework and let it finalize the session (and compute the process exit code) right away, so + // setting the flag afterwards would race that finalization and the deadline exit code could be + // missed. The graceful stop does not cancel the token, so unlike abort this flag is not set by a + // token-registered callback; set it here first. + await _policiesService.ExecuteDeadlineCallbacksAsync().ConfigureAwait(false); + + // Request the stop straight after the commit, with nothing awaited in between. Between the claim + // and this call the run is already committed as truncated while nothing has actually been asked to + // stop, and a completion arriving in that window can no longer disarm the deadline -- the claim + // closed that door. The stop would then find nothing left to stop, return successfully, and a run + // that executed every test would still exit with ExitCode.TestExecutionStoppedAtDeadline. Anything + // awaited here widens that window; the user-facing message used to sit in it and is bounded by + // _reportTimeout, so it alone could hold the window open for ten seconds. + await capability.StopTestExecutionAsync(_cancellationTokenSource.CancellationToken).ConfigureAwait(false); + stopAccepted = true; + + // Only now tell the user. It is written after the stop is accepted rather than before it for the + // window above, and it is reached only when the deadline actually won the race, so the message is + // never printed for a run that finished on its own or for a stop the framework rejected. The + // framework has been asked to stop but in-flight tests are still finishing, so this still lands + // before the end-of-run summary. + await TryReportAsync( + () => _outputDevice.DisplayAsync( + this, + new FormattedTextOutputDeviceData(PlatformResources.AbortAtDeadlineMessage), + _cancellationTokenSource.CancellationToken), + "Failed to report the approaching deadline.").ConfigureAwait(false); + } + catch (Exception ex) + { + // The verdict above was committed on the assumption that the stop would be accepted. It was not, + // so the framework was never asked to stop and the run carries on to completion -- leaving the + // verdict set would report TestExecutionStoppedAtDeadline for a run that executed every test. + // Take it back. This cannot resurrect the finalization race the early commit avoids: that race + // only exists once the stop is accepted, and here it never was. + // Best-effort: never let the timer callback crash the process during teardown. The error + // log is itself best-effort (the logger may be what threw), so guard it too. + try + { + await _logger.LogErrorAsync("Failed to request graceful stop at deadline.", ex).ConfigureAwait(false); + } + catch (Exception) + { + // Ignore: nothing else can be done here. + } + } + finally + { + if (!stopAccepted) + { + // The stop request was rejected (StopTestExecutionAsync is a [TPEXP] extensibility point, so a + // framework can throw anything), which means the platform truncated nothing. Take the outcome + // back, otherwise a run that carries on and executes every test still exits with + // ExitCode.TestExecutionStoppedAtDeadline and fails a job that actually completed. If the run + // is instead hard-killed at the real deadline, the process never reports an exit code at all, + // so reverting cannot mask a genuine truncation. + _policiesService.RevertDeadlineTriggered(); + ReleaseDeadlineClaim(); + } + } + } + + /// + /// Runs a best-effort diagnostic, swallowing anything it throws and giving up on it if it does not + /// complete promptly, so it can never skip or delay the graceful stop. + /// + private async Task TryReportAsync(Func report, string failureMessage) + { + try + { + // Swallowing faults is not enough on its own. A wedged logger or output device does not throw, + // it hands back a task that never completes. Before the claim that would stop the handler ever + // reaching the graceful stop, so the deadline would pass with nothing done; after the stop it + // would keep the handler task alive until disposal gave up on draining it. Bound the wait: + // TimeoutAfterAsync abandons the task and keeps observing it, so a fault arriving later cannot + // resurface as an unobserved task exception. + await report().TimeoutAfterAsync(_reportTimeout).ConfigureAwait(false); + } + catch (Exception ex) + { + // Even this failure log is best-effort, and bounded for the same reason: the logger may be + // exactly what threw or wedged, so reporting the failure must not re-throw or block the stop. + try + { + await _logger.LogErrorAsync(failureMessage, ex).TimeoutAfterAsync(_reportTimeout).ConfigureAwait(false); + } + catch (Exception) + { + // Ignore: the graceful stop is the only thing that must happen. + } + } + } + + public void Dispose() + { + // Capture the in-flight handler task under the lock so we either observe the task the timer + // callback published (and drain it below) or set _disposed first (so the callback never + // starts the handler). See OnDeadlineReached. + Task? handleDeadlineTask; + lock (_lock) + { + _disposed = true; + handleDeadlineTask = _handleDeadlineTask; + } + + _timer?.Dispose(); +#if !NETCOREAPP + // netstandard2.0 has no ValueTask/IAsyncDisposable, so there is no async drain path (see + // DisposeAsync below, which is netcoreapp-only). Fall back to a bounded blocking drain so an + // in-flight deadline handler can finish reporting before teardown, without letting a wedged + // graceful stop hang disposal. HandleDeadlineAsync swallows its own failures, so this wait + // never observes a fault. + try + { + handleDeadlineTask?.Wait(DisposeDrainTimeout); + } + catch (Exception) + { + // Best-effort drain: disposal must never throw. + } +#endif + } + +#if NETCOREAPP + public async ValueTask DisposeAsync() + { + // Capture the in-flight handler task under the lock so we either observe the task the timer + // callback published (and drain it below) or set _disposed first (so the callback never + // starts the handler). See OnDeadlineReached. + Task? handleTask; + lock (_lock) + { + _disposed = true; + handleTask = _handleDeadlineTask; + } + + _timer?.Dispose(); + + // Drain an in-flight deadline handler so its reporting can finish before the host tears down, + // but bound the wait so a wedged graceful stop cannot hang disposal. HandleDeadlineAsync + // swallows its own failures, so awaiting the completed task here never throws. + if (handleTask is not null) + { + Task completed = await Task.WhenAny(handleTask, Task.Delay(DisposeDrainTimeout)).ConfigureAwait(false); + if (completed == handleTask) + { + await handleTask.ConfigureAwait(false); + } + } + } +#endif + + /// + /// Which of test execution finishing and the deadline firing took the run. Transitions happen only out of + /// and only under the extension's lock, so the two are mutually exclusive. + /// + private enum RunState + { + /// + /// Test execution is in progress, so the deadline still applies. + /// + Running, + + /// + /// The test framework invoker returned: every test that was going to run has run, so a deadline + /// reached from here on must not mark the run as truncated. + /// + Completed, + + /// + /// The deadline fired while tests were still running and owns the verdict. Test execution completing + /// afterwards is the stop taking effect, so it must not take the verdict back. + /// + DeadlineClaimed, + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/DeadlineHelper.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/DeadlineHelper.cs new file mode 100644 index 0000000000..170bf43e47 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/DeadlineHelper.cs @@ -0,0 +1,83 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.CodeAnalysis; + +namespace Microsoft.Testing.Platform.Helpers; + +/// +/// Reads the CI-imposed hard-cancel deadline and its associated margins from the environment. +/// The deadline is exported by the CI system (or Arcade) as an absolute wall-clock instant so +/// that both the in-process test host and the out-of-process test host controller can schedule +/// their reactions (graceful stop, hang dump) backwards from the same instant. +/// +[Embedded] +internal static class DeadlineHelper +{ + // Prototype defaults. stopMargin > dumpMargin so the graceful stop is attempted first and the + // hang dump is the fallback for a host that did not stop in time. + private static readonly TimeSpan DefaultStopMargin = TimeSpan.FromSeconds(60); + private static readonly TimeSpan DefaultDumpMargin = TimeSpan.FromSeconds(30); + + /// + /// Attempts to read and parse + /// it as an absolute instant in UTC. + /// + public static bool TryGetDeadline(IEnvironment environment, out DateTimeOffset deadlineUtc) + { + deadlineUtc = default; + string? raw = environment.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE); + if (RoslynString.IsNullOrWhiteSpace(raw)) + { + return false; + } + + if (!DateTimeOffset.TryParse(raw, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out DateTimeOffset parsed)) + { + return false; + } + + deadlineUtc = parsed.ToUniversalTime(); + return true; + } + + /// + /// Gets the lead time before the deadline at which the platform should gracefully stop scheduling + /// new tests. Reads + /// (bare numbers are seconds); falls back to a default when unset or unparsable. + /// + public static TimeSpan GetStopMargin(IEnvironment environment) + => GetMargin(environment, EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN, DefaultStopMargin); + + /// + /// Gets the lead time before the deadline at which the platform should take a hang dump. Reads + /// (bare numbers + /// are seconds); falls back to a default when unset or unparsable. + /// + public static TimeSpan GetDumpMargin(IEnvironment environment) + => GetMargin(environment, EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN, DefaultDumpMargin); + + /// + /// Subtracts from , clamping the result at + /// instead of throwing when the subtraction would underflow. + /// A very old (but valid) deadline, or a large margin, could otherwise overflow while computing + /// the stop/dump instant. Saturating means "this instant is already in the past", which for both + /// callers translates to "act immediately". + /// + public static DateTimeOffset SubtractSaturating(DateTimeOffset instant, TimeSpan margin) + => margin > instant - DateTimeOffset.MinValue + ? DateTimeOffset.MinValue + : instant - margin; + + private static TimeSpan GetMargin(IEnvironment environment, string variableName, TimeSpan defaultValue) + { + string? raw = environment.GetEnvironmentVariable(variableName); + + // TimeSpanParser only matches non-negative numbers (its regex has no sign), so a parsed + // value is always >= zero; no extra sign check is needed here. + return !RoslynString.IsNullOrWhiteSpace(raw) + && TimeSpanParser.TryParse(raw, TimeSpanDefaultUnit.Seconds, out TimeSpan parsed) + ? parsed + : defaultValue; + } +} diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs index d42f5e3f41..409a008bc8 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/EnvironmentVariableConstants.cs @@ -90,4 +90,12 @@ internal static class EnvironmentVariableConstants // Trx public const string TESTINGPLATFORM_TRX_TESTRUN_ID = nameof(TESTINGPLATFORM_TRX_TESTRUN_ID); + + // Deadline-aware cancellation. TESTINGPLATFORM_DEADLINE is an absolute wall-clock instant + // (ISO 8601 round-trip, parsed to UTC) that the CI runner will hard-cancel the process at. + // The margins are the lead time before the deadline at which the platform reacts: graceful + // stop (stop scheduling new tests, let reporters finalize) and hang dump (out-of-proc dump). + public const string TESTINGPLATFORM_DEADLINE = nameof(TESTINGPLATFORM_DEADLINE); + public const string TESTINGPLATFORM_DEADLINE_STOP_MARGIN = nameof(TESTINGPLATFORM_DEADLINE_STOP_MARGIN); + public const string TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = nameof(TESTINGPLATFORM_DEADLINE_DUMP_MARGIN); } diff --git a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs index 521fa77e34..119c885b41 100644 --- a/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs +++ b/src/Platform/Microsoft.Testing.Platform/Helpers/ExitCodes.cs @@ -84,4 +84,9 @@ internal enum ExitCode /// One or more code-coverage thresholds were not met. /// CoverageThresholdFailed = 14, + + /// + /// Test execution stopped early because the configured deadline was approaching, so not every test ran. + /// + TestExecutionStoppedAtDeadline = 15, } diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.MessageBus.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.MessageBus.cs index 4ab4dca444..720862a46d 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.MessageBus.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/CommonTestHost.MessageBus.cs @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using Microsoft.Testing.Platform.Extensions; using Microsoft.Testing.Platform.Extensions.TestFramework; using Microsoft.Testing.Platform.Messages; using Microsoft.Testing.Platform.OutputDevice; @@ -34,7 +35,22 @@ protected static async Task ExecuteRequestAsync(ProxyOutputDevice outputDevice, using (otelService?.StartActivity("TestFrameworkInvoker")) { - await serviceProvider.GetTestFrameworkInvoker().ExecuteAsync(testFramework, client, cancellationToken).ConfigureAwait(false); + try + { + await serviceProvider.GetTestFrameworkInvoker().ExecuteAsync(testFramework, client, cancellationToken).ConfigureAwait(false); + } + finally + { + // Test execution is over -- normally, or because it failed or was canceled. Disarm the + // deadline here, before end-of-session draining and reporting begin, so a deadline + // reached while the reporters finalize an already-executed run cannot mark it as + // truncated. The extension cannot do this from ITestSessionLifetimeHandler: it is an + // IDataConsumer, and consumer handlers run last in NotifyTestSessionEndAsync, after the + // drains and after the reporters. Absent in server mode and for discovery, where the + // extension is not registered. + serviceProvider.GetService()?.NotifyTestExecutionCompleted(); + serviceProvider.GetRequiredService().NotifyTestExecutionCompleted(); + } } using (otelService?.StartActivity("OnTestSessionEnding")) diff --git a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Framework.cs b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Framework.cs index a2f5e36eb5..cda88960db 100644 --- a/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Framework.cs +++ b/src/Platform/Microsoft.Testing.Platform/Hosts/TestHostBuilder.Framework.cs @@ -90,8 +90,6 @@ private static async Task BuildTestFrameworkAsync(TestFrameworkB testSessionLifetimeHandlers.Add(pushOnlyProtocolDataConsumer); } - serviceProvider.AddService(new TestSessionLifetimeHandlersContainer(testSessionLifetimeHandlers)); - ITestApplicationProcessExitCode testApplicationResult = serviceProvider.GetRequiredService(); await RegisterAsServiceOrConsumerOrBothAsync(testApplicationResult, serviceProvider, dataConsumersBuilder).ConfigureAwait(false); @@ -114,6 +112,42 @@ private static async Task BuildTestFrameworkAsync(TestFrameworkB dataConsumersBuilder.Add(abortForMaxFailedTestsExtension); } + // Deadline-aware graceful stop only makes sense for a console execution run. Skip it for + // server mode (BuildTestFrameworkAsync runs per request, which would re-arm the timer against + // an already-past deadline and fire on every request) and for discovery-only requests. + if (pushOnlyProtocol?.IsServerMode != true && !testFrameworkBuilderData.IsForDiscoveryRequest) + { + var abortAtDeadlineExtension = new AbortAtDeadlineExtension( + serviceProvider.GetEnvironment(), + serviceProvider.GetSystemClock(), + serviceProvider.GetTestFrameworkCapabilities().GetCapability(), + serviceProvider.GetRequiredService(), + serviceProvider.GetTestApplicationCancellationTokenSource(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetLoggerFactory()); + + if (await abortAtDeadlineExtension.IsEnabledAsync().ConfigureAwait(false)) + { + dataConsumersBuilder.Add(abortAtDeadlineExtension); + + // Also register it as a service so the host can tell it, the moment the test framework invoker + // returns, that test execution is over. On that signal it disarms the deadline, so a timer + // firing while the reporters finalize an already-finished run cannot wrongly mark the run as + // deadline-truncated (exit code 15). A session-lifetime handler would be too late: this is an + // IDataConsumer, and consumer handlers run at the very end of NotifyTestSessionEndAsync. + serviceProvider.AddService(abortAtDeadlineExtension); + + // Keep the lifetime-handler registration as a backstop for host paths that do not execute the + // invoker path above. It runs too late to protect reporting on its own. + testSessionLifetimeHandlers.Add(abortAtDeadlineExtension); + } + } + + // The container captures the list by reference (so a later Add would still be observed), but populating + // it fully before registering keeps this order-independent and free of that subtlety. Lifetime handlers + // are enumerated later, during NotifyTestSessionEndAsync. + serviceProvider.AddService(new TestSessionLifetimeHandlersContainer(testSessionLifetimeHandlers)); + AsynchronousMessageBus concreteMessageBusService = new( [.. dataConsumersBuilder], serviceProvider.GetTestApplicationCancellationTokenSource(), diff --git a/src/Platform/Microsoft.Testing.Platform/InternalAPI/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/InternalAPI/InternalAPI.Unshipped.txt index 179cd501a2..f06328a29b 100644 --- a/src/Platform/Microsoft.Testing.Platform/InternalAPI/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/InternalAPI/InternalAPI.Unshipped.txt @@ -150,6 +150,19 @@ Microsoft.Testing.Platform.OutputDevice.Terminal.TestProgressState.GetOrCreateTe Microsoft.Testing.Platform.OutputDevice.Terminal.TestProgressState.NotifyHandshake(string! instanceId, int attemptNumber) -> void Microsoft.Testing.Platform.OutputDevice.Terminal.TestProgressState.ReportDiscoveredTest(string? displayName) -> void Microsoft.Testing.Platform.Helpers.ExitCode.CoverageThresholdFailed = 14 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Helpers.ExitCode.TestExecutionStoppedAtDeadline = 15 -> Microsoft.Testing.Platform.Helpers.ExitCode +Microsoft.Testing.Platform.Services.IStopPoliciesService.IsDeadlineTriggered.get -> bool +Microsoft.Testing.Platform.Services.IStopPoliciesService.RegisterOnDeadlineCallbackAsync(System.Func! callback) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Services.IStopPoliciesService.ExecuteDeadlineCallbacksAsync() -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Services.IStopPoliciesService.IsTestExecutionCompleted.get -> bool +Microsoft.Testing.Platform.Services.IStopPoliciesService.NotifyTestExecutionCompleted() -> void +Microsoft.Testing.Platform.Services.IStopPoliciesService.RevertDeadlineTriggered() -> void +Microsoft.Testing.Platform.Services.StopPoliciesService.IsTestExecutionCompleted.get -> bool +Microsoft.Testing.Platform.Services.StopPoliciesService.NotifyTestExecutionCompleted() -> void +Microsoft.Testing.Platform.Services.StopPoliciesService.RevertDeadlineTriggered() -> void +Microsoft.Testing.Platform.Services.StopPoliciesService.IsDeadlineTriggered.get -> bool +Microsoft.Testing.Platform.Services.StopPoliciesService.RegisterOnDeadlineCallbackAsync(System.Func! callback) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Services.StopPoliciesService.ExecuteDeadlineCallbacksAsync() -> System.Threading.Tasks.Task! Microsoft.Testing.Platform.OutputDevice.Terminal.TerminalTestReporter.AppendCoverageSummary(System.Collections.Generic.IReadOnlyList! scopes, System.Collections.Generic.IReadOnlyList! thresholds) -> void Microsoft.Testing.Platform.OutputDevice.TerminalOutputDevice.TerminalOutputDevice(Microsoft.Testing.Platform.Helpers.IConsole! console, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.TestHostControllers.ITestHostControllerInfo! testHostControllerInfo, Microsoft.Testing.Platform.Helpers.IAsyncMonitor! asyncMonitor, Microsoft.Testing.Platform.Helpers.IRuntimeFeature! runtimeFeature, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Services.IPlatformInformation! platformInformation, Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Logging.IFileLoggerInformation? fileLoggerInformation, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory, Microsoft.Testing.Platform.Helpers.IClock! clock, Microsoft.Testing.Platform.Services.IStopPoliciesService! policiesService, Microsoft.Testing.Platform.Services.ITestApplicationCancellationTokenSource! testApplicationCancellationTokenSource, Microsoft.Testing.Platform.Services.ITestCoverageResult! testCoverageResult) -> void *REMOVED*Microsoft.Testing.Platform.OutputDevice.TerminalOutputDevice.TerminalOutputDevice(Microsoft.Testing.Platform.Helpers.IConsole! console, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.TestHostControllers.ITestHostControllerInfo! testHostControllerInfo, Microsoft.Testing.Platform.Helpers.IAsyncMonitor! asyncMonitor, Microsoft.Testing.Platform.Helpers.IRuntimeFeature! runtimeFeature, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Services.IPlatformInformation! platformInformation, Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions, Microsoft.Testing.Platform.Logging.IFileLoggerInformation? fileLoggerInformation, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory, Microsoft.Testing.Platform.Helpers.IClock! clock, Microsoft.Testing.Platform.Services.IStopPoliciesService! policiesService, Microsoft.Testing.Platform.Services.ITestApplicationCancellationTokenSource! testApplicationCancellationTokenSource) -> void @@ -308,6 +321,27 @@ static Microsoft.Testing.Platform.OutputDevice.TerminalOutputDevice.GetSlowestTe *REMOVED*static Microsoft.Testing.Platform.Services.ServiceProviderExtensions.GetSystemClock(this System.IServiceProvider! serviceProvider) -> Microsoft.Testing.Platform.Helpers.IClock! *REMOVED*virtual Microsoft.Testing.Platform.CommandLine.CommandLineOptionsProviderBase.ValidateCommandLineOptionsAsync(Microsoft.Testing.Platform.CommandLine.ICommandLineOptions! commandLineOptions) -> System.Threading.Tasks.Task! *REMOVED*virtual Microsoft.Testing.Platform.CommandLine.CommandLineOptionsProviderBase.ValidateOptionArgumentsAsync(Microsoft.Testing.Platform.Extensions.CommandLine.CommandLineOption! commandOption, string![]! arguments) -> System.Threading.Tasks.Task! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE = "TESTINGPLATFORM_DEADLINE" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN = "TESTINGPLATFORM_DEADLINE_STOP_MARGIN" -> string! +const Microsoft.Testing.Platform.Helpers.EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN = "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN" -> string! +Microsoft.Testing.Platform.Helpers.DeadlineHelper +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.TryGetDeadline(Microsoft.Testing.Platform.Helpers.IEnvironment! environment, out System.DateTimeOffset deadlineUtc) -> bool +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.GetStopMargin(Microsoft.Testing.Platform.Helpers.IEnvironment! environment) -> System.TimeSpan +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.GetDumpMargin(Microsoft.Testing.Platform.Helpers.IEnvironment! environment) -> System.TimeSpan +static Microsoft.Testing.Platform.Helpers.DeadlineHelper.SubtractSaturating(System.DateTimeOffset instant, System.TimeSpan margin) -> System.DateTimeOffset +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.AbortAtDeadlineExtension(Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Helpers.IClock! clock, Microsoft.Testing.Platform.Capabilities.TestFramework.IGracefulStopTestExecutionCapability? capability, Microsoft.Testing.Platform.Services.IStopPoliciesService! policiesService, Microsoft.Testing.Platform.Services.ITestApplicationCancellationTokenSource! cancellationTokenSource, Microsoft.Testing.Platform.OutputDevice.IOutputDevice! outputDevice, Microsoft.Testing.Platform.Logging.ILoggerFactory! loggerFactory, System.TimeSpan? reportTimeout = null) -> void +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.ConsumeAsync(Microsoft.Testing.Platform.Extensions.Messages.IDataProducer! dataProducer, Microsoft.Testing.Platform.Extensions.Messages.IData! value, System.Threading.CancellationToken cancellationToken) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.DataTypesConsumed.get -> System.Type![]! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.Description.get -> string! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.DisplayName.get -> string! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.Dispose() -> void +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.IsEnabledAsync() -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.NotifyTestExecutionCompleted() -> void +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.OnTestSessionFinishingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! testSessionContext) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.OnTestSessionStartingAsync(Microsoft.Testing.Platform.Services.ITestSessionContext! testSessionContext) -> System.Threading.Tasks.Task! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.Uid.get -> string! +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.Version.get -> string! Microsoft.Testing.Platform.DotnetTestConnection.DotnetTestConnection(Microsoft.Testing.Platform.CommandLine.CommandLineHandler! commandLineHandler, Microsoft.Testing.Platform.Helpers.IEnvironment! environment, Microsoft.Testing.Platform.Services.ITestApplicationModuleInfo! testApplicationModuleInfo, Microsoft.Testing.Platform.Services.ITestApplicationCancellationTokenSource! cancellationTokenSource, Microsoft.Testing.Platform.Logging.ILogger! logger) -> void Microsoft.Testing.Platform.OutputDevice.Terminal.TerminalTestReporter.TerminalTestReporter(Microsoft.Testing.Platform.Helpers.IConsole! console, System.Func! isCancellationRequested, Microsoft.Testing.Platform.OutputDevice.Terminal.TerminalTestReporterOptions! options, Microsoft.Testing.Platform.Logging.ILogger! logger) -> void Microsoft.Testing.Platform.OutputDevice.Terminal.TestProgressStateAwareTerminal.TestProgressStateAwareTerminal(Microsoft.Testing.Platform.OutputDevice.Terminal.ITerminal! terminal, System.Func! showProgress, Microsoft.Testing.Platform.OutputDevice.Terminal.IProgressRenderer! renderer, Microsoft.Testing.Platform.Logging.ILogger! logger) -> void diff --git a/src/Platform/Microsoft.Testing.Platform/InternalAPI/net/InternalAPI.Unshipped.txt b/src/Platform/Microsoft.Testing.Platform/InternalAPI/net/InternalAPI.Unshipped.txt index 0fee2424cc..3740e04415 100644 --- a/src/Platform/Microsoft.Testing.Platform/InternalAPI/net/InternalAPI.Unshipped.txt +++ b/src/Platform/Microsoft.Testing.Platform/InternalAPI/net/InternalAPI.Unshipped.txt @@ -1,4 +1,5 @@ #nullable enable +Microsoft.Testing.Platform.Extensions.AbortAtDeadlineExtension.DisposeAsync() -> System.Threading.Tasks.ValueTask Microsoft.Testing.Platform.Messages.AsyncConsumerDataProcessor.AsyncConsumerDataProcessor(Microsoft.Testing.Platform.Extensions.IDataConsumer! consumer, Microsoft.Testing.Platform.Helpers.ITask! task, System.Threading.CancellationToken cancellationToken, System.TimeSpan canceledShutdownTimeout) -> void *REMOVED*Microsoft.Testing.Platform.ServerMode.FormatterUtilities.MessageFormatter.Deserialize(System.ReadOnlyMemory serializedUtf8Content) -> T *REMOVED*Microsoft.Testing.Platform.ServerMode.IMessageFormatter.Deserialize(System.ReadOnlyMemory serializedUtf8Content) -> T diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx index 2cb83fef9c..314a72f435 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx +++ b/src/Platform/Microsoft.Testing.Platform/Resources/PlatformResources.resx @@ -950,6 +950,12 @@ Valid values are 'allow-skipped' (the default) which counts skipped tests as run Extension used to support '--maximum-failed-tests'. When a given failures threshold is reached, the test run will be aborted. {Locked="--maximum-failed-tests"} + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Test session is aborting due to reaching failures ('{0}') specified by the '--maximum-failed-tests' option. {0} is the number of max failed tests. {Locked="--maximum-failed-tests"} diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf index 0570ad14d0..5ecfa8683b 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.cs.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. Aktuální testovací architektura neimplementuje rozhraní IGracefulStopTestExecutionCapability, které je vyžadováno pro funkci --maximum-failed-tests. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf index 57fd422026..83773d1fc6 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.de.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. Das aktuelle Testframework implementiert nicht "IGracefulStopTestExecutionCapability", das für das Feature "--maximum-failed-tests" erforderlich ist. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf index de51f08be2..95f3495e28 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.es.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. El marco de pruebas actual no implementa "IGracefulStopTestExecutionCapability", que es necesario para la característica "--maximum-failed-tests". diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf index 4a5999c554..998ff3d924 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.fr.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. Le framework de tests actuel n’implémente pas 'IGracefulStopTestExecutionCapability', qui est requis pour la fonctionnalité '--maximum-failed-tests'. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf index 7187563fcf..7b5ec74ec7 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.it.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. Il framework di test corrente non implementa 'IGracefulStopTestExecutionCapability', necessario per la funzionalità '--maximum-failed-tests'. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf index 141943643b..9fd1766c21 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ja.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. 現在のテスト フレームワークは、'--maximum-failed-tests' 機能に必要な 'IGracefulStopTestExecutionCapability' を実装していません。 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf index 5485ef7d36..c48313dedc 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ko.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. 현재 테스트 프레임워크는 '--maximum-failed-tests' 기능에 필요한 'IGracefulStopTestExecutionCapability'를 구현하지 않습니다. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf index e9b5cdd772..742beeee17 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pl.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. Bieżąca platforma testowa nie implementuje interfejsu "IGracefulStopTestExecutionCapability", który jest wymagany dla funkcji "--maximum-failed-tests". diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf index fe55d3820c..b9eb08a865 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.pt-BR.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. A estrutura de teste atual não implementa 'IGracefulStopTestExecutionCapability', que é necessário para o recurso '--maximum-failed-tests'. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf index 51283d4997..4339944b41 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.ru.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. Текущая платформа тестирования не реализует параметр "IGracefulStopTestExecutionCapability", необходимый для функции "--maximum-failed-tests". diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf index 08e82bcc98..d501dcf9ad 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.tr.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. Geçerli test çerçevesi, '--maximum-failed-tests' özelliği için gerekli olan 'IGracefulStopTestExecutionCapability' gerçekleştiremiyor. diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf index 1dbd75f0cd..8f21169abc 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hans.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. 当前测试框架未实现 “--maximum-failed-tests” 功能所需的 “IGracefulStopTestExecutionCapability”。 diff --git a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf index a9997622e8..f8b99fec9d 100644 --- a/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Platform/Resources/xlf/PlatformResources.zh-Hant.xlf @@ -2,6 +2,16 @@ + + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + Extension that gracefully stops the test run shortly before a CI-imposed deadline so reports can be finalized. + + + + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + Deadline approaching: gracefully stopping the test run so reports can be finalized before the CI hard-cancel. + + The current test framework does not implement 'IGracefulStopTestExecutionCapability' which is required for '--maximum-failed-tests' feature. 目前的測試架構未實作 '--maximum-failed-tests' 功能所需的 'IGracefulStopTestExecutionCapability'。 diff --git a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs index d82def8818..c3d0f9b8fe 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/IStopPoliciesService.cs @@ -9,11 +9,46 @@ internal interface IStopPoliciesService bool IsAbortTriggered { get; } + bool IsDeadlineTriggered { get; } + + /// + /// Gets a value indicating whether test execution has finished, meaning the test framework invoker has + /// returned and only reporting and teardown remain. + /// + /// + /// This is the gate that stops a deadline elapsing during reporting from marking an already-finished run as + /// deadline-truncated. It is monotonic: it only ever goes from to + /// , so it can be read without synchronization. + /// + bool IsTestExecutionCompleted { get; } + + /// + /// Records that test execution has finished. Called by the host the moment the test framework invoker + /// returns, before any reporting or message-bus draining starts. + /// + void NotifyTestExecutionCompleted(); + + /// + /// Clears the deadline-triggered outcome recorded by . + /// + /// + /// The deadline outcome has to be recorded before the graceful stop is requested, because the stop can let + /// the framework finalize the session (and compute the exit code) immediately. When the stop request is then + /// rejected, nothing was truncated on the platform's account, so the outcome has to be taken back: otherwise + /// a run that goes on to execute every test still reports + /// . + /// + void RevertDeadlineTriggered(); + Task RegisterOnMaxFailedTestsCallbackAsync(Func callback); Task RegisterOnAbortCallbackAsync(Func callback); + Task RegisterOnDeadlineCallbackAsync(Func callback); + Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken); Task ExecuteAbortCallbacksAsync(); + + Task ExecuteDeadlineCallbacksAsync(); } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs index be2f89dfd7..14a6b3a544 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/StopPoliciesService.cs @@ -11,8 +11,36 @@ internal sealed class StopPoliciesService : IStopPoliciesService private readonly ConcurrentQueue> _maxFailedTestsCallbacks = new(); private readonly ConcurrentQueue> _abortCallbacks = new(); + + // Guards the deadline state together with its callback list, so registration and the one-shot trigger + // cannot interleave and drop a callback. A flag plus a concurrent queue is not enough: the registering + // thread can read the flag as false, the trigger can then set it and snapshot a still-empty queue, and only + // afterwards does the callback land in the queue -- where nothing will ever invoke it, because the deadline + // fires once. Under this lock a callback is invoked exactly once, either by the trigger (it was in the list + // when the snapshot was taken) or by the registering thread itself (the trigger had already happened). +#if NET9_0_OR_GREATER + private readonly Lock _deadlineLock = new(); +#else + private readonly object _deadlineLock = new(); +#endif + private readonly List> _deadlineCallbacks = []; + + // Whether the callbacks have run. This is the one-shot gate and it is never cleared: once the callbacks + // have run, a second trigger must not run them again and a late registration must be invoked on the spot. + private bool _areDeadlineCallbacksExecuted; + + // Whether the run is to be reported as stopped at the deadline. This is only the exit-code verdict, and + // RevertDeadlineTriggered clears it when the graceful stop it was meant to precede could not be requested. + // Kept separate from _areDeadlineCallbacksExecuted so reverting the verdict cannot re-arm the callbacks. +#pragma warning disable IDE0032 // Use auto property - synchronized access requires a backing field. + private bool _isDeadlineTriggered; +#pragma warning restore IDE0032 private int _lastMaxFailedTests; + // Read on the deadline timer thread and written on the host thread that runs the request, so it is + // volatile. It is monotonic (false -> true only), so no lock is needed to read it consistently. + private volatile bool _isTestExecutionCompleted; + public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicationCancellationTokenSource) { _testApplicationCancellationTokenSource = testApplicationCancellationTokenSource; @@ -29,6 +57,21 @@ public StopPoliciesService(ITestApplicationCancellationTokenSource testApplicati public bool IsAbortTriggered { get; private set; } + public bool IsDeadlineTriggered + { + get + { + lock (_deadlineLock) + { + return _isDeadlineTriggered; + } + } + } + + public bool IsTestExecutionCompleted => _isTestExecutionCompleted; + + public void NotifyTestExecutionCompleted() => _isTestExecutionCompleted = true; + public async Task ExecuteMaxFailedTestsCallbacksAsync(int maxFailedTests, CancellationToken cancellationToken) { _lastMaxFailedTests = maxFailedTests; @@ -63,6 +106,47 @@ public async Task ExecuteAbortCallbacksAsync() } } + public async Task ExecuteDeadlineCallbacksAsync() + { + Func[] callbacks; + lock (_deadlineLock) + { + if (_areDeadlineCallbacksExecuted) + { + // The deadline is one-shot; a second trigger must not run the callbacks again. This is + // gated on the callback flag rather than the verdict, so a reverted verdict cannot let a + // later trigger run them a second time. + return; + } + + _areDeadlineCallbacksExecuted = true; + _isDeadlineTriggered = true; + + // Take the callbacks under the lock and clear the list, so a callback registered from now on is + // invoked by RegisterOnDeadlineCallbackAsync instead of being silently dropped here. + callbacks = [.. _deadlineCallbacks]; + _deadlineCallbacks.Clear(); + } + + foreach (Func callback in callbacks) + { + // For now, we are fine if the callback crashed us. It shouldn't happen for our + // current usage anyway and the APIs around this are all internal for now. + await callback.Invoke().ConfigureAwait(false); + } + } + + public void RevertDeadlineTriggered() + { + lock (_deadlineLock) + { + // Only the verdict is cleared. _areDeadlineCallbacksExecuted deliberately stays set: the + // callbacks have already run, and re-arming them here would let a later trigger run them a + // second time and would queue a late registration into a list nothing drains any more. + _isDeadlineTriggered = false; + } + } + public async Task RegisterOnMaxFailedTestsCallbackAsync(Func callback) { if (ProcessRole != TestProcessRole.TestHost) @@ -87,4 +171,21 @@ public async Task RegisterOnAbortCallbackAsync(Func callback) _abortCallbacks.Enqueue(callback); } + + public async Task RegisterOnDeadlineCallbackAsync(Func callback) + { + lock (_deadlineLock) + { + if (!_areDeadlineCallbacksExecuted) + { + _deadlineCallbacks.Add(callback); + return; + } + } + + // The callbacks already ran, so this registration came too late for the snapshot in + // ExecuteDeadlineCallbacksAsync. Invoke the callback here instead, outside the lock: it is + // arbitrary code and must not run while the deadline transition is held. + await callback().ConfigureAwait(false); + } } diff --git a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs index 13a59de88a..e2240baa57 100644 --- a/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs +++ b/src/Platform/Microsoft.Testing.Platform/Services/TestApplicationResult.cs @@ -175,6 +175,13 @@ internal int GetProcessExitCodeWithoutIgnore() exitCode = exitCode == ExitCode.Success && _failedTestsCount > 0 ? ExitCode.AtLeastOneTestFailed : exitCode; exitCode = exitCode == ExitCode.Success && _policiesService.IsAbortTriggered ? ExitCode.TestSessionAborted : exitCode; + // A deadline-driven graceful stop (see AbortAtDeadlineExtension) truncates the run before the CI + // hard-cancel. Such a run may otherwise look successful (it did not fail or abort), but it did not + // execute every test, so it must not report success. Real failures/abort above keep precedence; a + // clean-but-truncated run becomes non-zero here and takes precedence over the zero-tests/coverage + // verdicts below (a truncated run legitimately may not have run the expected number of tests). + exitCode = exitCode == ExitCode.Success && _policiesService.IsDeadlineTriggered ? ExitCode.TestExecutionStoppedAtDeadline : exitCode; + // An explicitly-provided `--minimum-expected-tests` governs the count-based verdict and // supersedes the ZeroTests (8) verdict below: a run of fewer than N tests yields // ExitCode.MinimumExpectedTestsPolicyViolation (9), even when zero tests ran. This lets callers diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AbortAtDeadlineTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AbortAtDeadlineTests.cs new file mode 100644 index 0000000000..353b4b1d01 --- /dev/null +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AbortAtDeadlineTests.cs @@ -0,0 +1,231 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Platform.Acceptance.IntegrationTests; + +[TestClass] +public sealed class AbortAtDeadlineTests : AcceptanceTestBase +{ + private const string AssetName = nameof(AbortAtDeadlineTests); + + private const string StopMessage = "gracefully stopping the test run so reports can be finalized"; + + [TestMethod] + public async Task WhenDeadlineIsInThePast_GracefullyStopsImmediately() + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + TestHostResult testHostResult = await testHost.ExecuteAsync( + environmentVariables: new() + { + // A deadline already in the past means the stop instant is also in the past, so the + // graceful stop fires as soon as the extension arms its timer. + ["TESTINGPLATFORM_DEADLINE"] = DateTimeOffset.UtcNow.AddMinutes(-5).ToString("o"), + ["TESTINGPLATFORM_DEADLINE_STOP_MARGIN"] = "0", + ["WAIT_FOR_STOP"] = "1", + }, + cancellationToken: TestContext.CancellationToken); + + // A deadline-truncated run reports its own exit code so CI/tooling can tell it apart from a + // clean pass, even though the in-flight test finished and the summary shows it as passed. + testHostResult.AssertExitCodeIs(ExitCode.TestExecutionStoppedAtDeadline); + testHostResult.AssertOutputContains(StopMessage); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); + } + + [TestMethod] + public async Task WhenDeadlineIsInTheFuture_GracefullyStopsWhenReached() + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + TestHostResult testHostResult = await testHost.ExecuteAsync( + environmentVariables: new() + { + // Stop margin 0 means the graceful stop is scheduled for the deadline itself, a few + // seconds out. The framework blocks until the stop is requested, so this proves the + // timer fires on schedule (not only when the deadline is already past). + ["TESTINGPLATFORM_DEADLINE"] = DateTimeOffset.UtcNow.AddSeconds(6).ToString("o"), + ["TESTINGPLATFORM_DEADLINE_STOP_MARGIN"] = "0", + ["WAIT_FOR_STOP"] = "1", + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.TestExecutionStoppedAtDeadline); + testHostResult.AssertOutputContains(StopMessage); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); + } + + [TestMethod] + public async Task WhenStopMarginIsSubtracted_GracefullyStopsBeforeDeadline() + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + TestHostResult testHostResult = await testHost.ExecuteAsync( + environmentVariables: new() + { + // Deadline is a minute out, but a 60s stop margin pulls the stop instant back to + // roughly now, exercising the margin subtraction against the absolute deadline. + ["TESTINGPLATFORM_DEADLINE"] = DateTimeOffset.UtcNow.AddSeconds(60).ToString("o"), + ["TESTINGPLATFORM_DEADLINE_STOP_MARGIN"] = "60", + ["WAIT_FOR_STOP"] = "1", + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.TestExecutionStoppedAtDeadline); + testHostResult.AssertOutputContains(StopMessage); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); + } + + [TestMethod] + public async Task WhenNoDeadlineIsSet_DoesNotStop() + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + // No deadline environment variable, and the framework does not wait for a stop, so the run + // completes normally and the extension stays silent (it is strictly opt-in). + TestHostResult testHostResult = await testHost.ExecuteAsync(cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + testHostResult.AssertOutputDoesNotContain(StopMessage); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); + } + + [TestMethod] + public async Task WhenGracefulStopCapabilityIsMissing_DoesNotStopAndRunsToCompletion() + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, TargetFrameworks.NetCurrent); + + // A deadline is set, but the framework does not expose IGracefulStopTestExecutionCapability, + // so the extension degrades to a no-op instead of failing the command line. + TestHostResult testHostResult = await testHost.ExecuteAsync( + environmentVariables: new() + { + ["TESTINGPLATFORM_DEADLINE"] = DateTimeOffset.UtcNow.AddMinutes(-5).ToString("o"), + ["DO_NOT_ADD_CAPABILITY"] = "1", + }, + cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.Success); + testHostResult.AssertOutputDoesNotContain(StopMessage); + testHostResult.AssertOutputContainsSummary(failed: 0, passed: 1, skipped: 0); + } + + public sealed class TestAssetFixture() : TestAssetFixtureBase() + { + private const string Sources = """ +#file AbortAtDeadlineTests.csproj + + + $TargetFrameworks$ + Exe + true + enable + preview + + + + + + +#file Program.cs +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Testing.Platform.Builder; +using Microsoft.Testing.Platform.Capabilities; +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.TestFramework; + +internal sealed class Program +{ + public static async Task Main(string[] args) + { + ITestApplicationBuilder builder = await TestApplication.CreateBuilderAsync(args); + builder.RegisterTestFramework(_ => new Capabilities(), (_, __) => new DummyTestFramework()); + using ITestApplication app = await builder.BuildAsync(); + return await app.RunAsync(); + } +} + +internal class DummyTestFramework : ITestFramework, IDataProducer +{ + public string Uid => nameof(DummyTestFramework); + + public string Version => string.Empty; + + public string DisplayName => string.Empty; + + public string Description => string.Empty; + + public Type[] DataTypesProduced => new[] { typeof(TestNodeUpdateMessage) }; + + public Task CloseTestSessionAsync(CloseTestSessionContext context) => Task.FromResult(new CloseTestSessionResult() { IsSuccess = true }); + + public Task CreateTestSessionAsync(CreateTestSessionContext context) => Task.FromResult(new CreateTestSessionResult() { IsSuccess = true }); + + public async Task ExecuteRequestAsync(ExecuteRequestContext context) + { + await context.MessageBus.PublishAsync(this, new TestNodeUpdateMessage(context.Request.Session.SessionUid, + new TestNode() { Uid = "1", DisplayName = "Test1", Properties = new(PassedTestNodeStateProperty.CachedInstance) })); + + // When asked to, block until the deadline-driven graceful stop is requested. This mimics a + // long-running suite whose remaining tests are cut short by the approaching CI deadline. Cap + // the wait so a broken stop path fails the assertions fast instead of hanging until the harness + // times out; the StopMessage and exit-code assertions still catch a stop that never happened. + if (Environment.GetEnvironmentVariable("WAIT_FOR_STOP") == "1") + { + await Task.WhenAny(GracefulStop.Instance.TCS.Task, Task.Delay(TimeSpan.FromMinutes(2))); + } + + context.Complete(); + } + + public Task IsEnabledAsync() => Task.FromResult(true); +} + +internal class Capabilities : ITestFrameworkCapabilities +{ + IReadOnlyCollection ICapabilities.Capabilities + { + get + { + if (Environment.GetEnvironmentVariable("DO_NOT_ADD_CAPABILITY") == "1") + { + return []; + } + + return [GracefulStop.Instance]; + } + } +} + +#pragma warning disable TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +internal sealed class GracefulStop : IGracefulStopTestExecutionCapability +#pragma warning restore TPEXP // Type is for evaluation purposes only and is subject to change or removal in future updates. Suppress this diagnostic to proceed. +{ + private GracefulStop() + { + } + + public static GracefulStop Instance { get; } = new(); + + public TaskCompletionSource TCS { get; } = new(); + + public Task StopTestExecutionAsync(CancellationToken cancellationToken) + { + TCS.TrySetResult(); + return Task.CompletedTask; + } +} + +"""; + + public string TargetAssetPath => GetAssetPath(AssetName); + + public override (string ID, string Name, string Code) GetAssetsToGenerate() => (AssetName, AssetName, + Sources + .PatchTargetFrameworks(TargetFrameworks.NetCurrent) + .PatchCodeWithReplace("$MicrosoftTestingPlatformVersion$", MicrosoftTestingPlatformVersion)); + } + + public TestContext TestContext { get; set; } +} diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HangDumpTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HangDumpTests.cs index 427fbaceae..09ef30714b 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HangDumpTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HangDumpTests.cs @@ -25,6 +25,31 @@ public async Task HangDump_DefaultSetting_CreateDump(string tfm) Assert.ContainsSingle(dumpFiles, $"Expected single dump file. Found: {Environment.NewLine}{string.Join(Environment.NewLine, dumpFiles)}{Environment.NewLine}{testHostResult}"); } + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task HangDump_AbsoluteDeadline_CreateDump(string tfm) + { + string resultDirectory = Path.Combine(AssetFixture.TargetAssetPath, Guid.NewGuid().ToString("N"), tfm); + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, "HangDump", tfm); + + // Inactivity timeout is huge so the classic hang path never fires. The dump is driven purely + // by the absolute CI deadline: a few seconds out, with a zero dump margin so it triggers at + // the deadline itself. The test hangs (SLEEPTIMEMS2), so the deadline is what takes the dump. + TestHostResult testHostResult = await testHost.ExecuteAsync( + $"--hangdump --hangdump-timeout 30m --results-directory {resultDirectory}", + new Dictionary + { + { "SLEEPTIMEMS1", "1000" }, + { "SLEEPTIMEMS2", "600000" }, + { "TESTINGPLATFORM_DEADLINE", DateTimeOffset.UtcNow.AddSeconds(8).ToString("o") }, + { "TESTINGPLATFORM_DEADLINE_DUMP_MARGIN", "0" }, + }, + cancellationToken: TestContext.CancellationToken); + testHostResult.AssertExitCodeIs(ExitCode.TestHostProcessExitedNonGracefully); + string[] dumpFiles = Directory.GetFiles(resultDirectory, "HangDump*.dmp", SearchOption.AllDirectories); + Assert.ContainsSingle(dumpFiles, $"Expected single dump file. Found: {Environment.NewLine}{string.Join(Environment.NewLine, dumpFiles)}{Environment.NewLine}{testHostResult}"); + } + [TestMethod] public async Task HangDump_WithDotnetTest_CreateDump() { diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs index 208ec58f35..41f2d231f0 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/GitHubActionsExitCodeTests.cs @@ -56,6 +56,16 @@ public void GetReason_ForKnownCode_MentionsRelevantOption() Assert.Contains("--minimum-expected-tests", GitHubActionsExitCode.GetReason(9)); Assert.Contains("--maximum-failed-tests", GitHubActionsExitCode.GetReason(13)); Assert.Contains("coverage threshold", GitHubActionsExitCode.GetReason(14)); + Assert.Contains("deadline", GitHubActionsExitCode.GetReason(15)); + } + + [TestMethod] + public void GetReason_ForDeadlineStop_IsNotTheUnknownFallback() + { + // 15 is the newest arm, so it is the one most likely to be dropped and silently answered by the + // unknown fallback. Pin it to its own text so that regression is visible. + Assert.AreNotEqual(GitHubActionsExitCode.GetReason(255), GitHubActionsExitCode.GetReason(15)); + Assert.AreEqual("TestExecutionStoppedAtDeadline", GitHubActionsExitCode.GetName(15)); } [TestMethod] diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HangDumpTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HangDumpTests.cs index 2cf0ceaab0..27f93d3167 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HangDumpTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/HangDumpTests.cs @@ -18,6 +18,8 @@ namespace Microsoft.Testing.Extensions.UnitTests; [TestClass] public sealed class HangDumpTests { + public TestContext TestContext { get; set; } = null!; + private HangDumpCommandLineProvider GetProvider() { var testApplicationModuleInfo = new Mock(); @@ -140,6 +142,117 @@ public void GetDumpFileNames_WindowsPathWithSpaces_QuotesOnlyWriteDumpArgument() Assert.AreEqual(dumpFileName, dumpFileNames.ArtifactDumpFileName); } + [TestMethod] + public async Task QueryOnceAndDumpTree_WithStalledQuery_QueriesOncePerDumpAndStillDumpsWholeTree() + { + // A wedged test host never answers the in-progress-test query, so the query costs a full + // InProgressTestsQueryTimeout. Issuing it per process would multiply that bound by the size of + // the tree, so a six-process tree must still pay it exactly once and then dump every process. + int queryCount = 0; + IProcess[] bottomUpTree = [.. Enumerable.Range(0, 6).Select(_ => Mock.Of())]; + List dumped = []; + List<(string, int)[]> annotations = []; + + await HangDumpProcessLifetimeHandler.QueryOnceAndDumpTreeAsync( + bottomUpTree, + cancellationToken => + { + Interlocked.Increment(ref queryCount); + + // The real bounded query, against a reply that never arrives: the product's own bound + // cancels the wait and the dump proceeds with an empty list. + return HangDumpProcessLifetimeHandler.QueryInProgressTestsWithTimeoutAsync( + async queryCancellationToken => + { + await Task.Delay(Timeout.Infinite, queryCancellationToken); + return []; + }, + TimeSpan.FromMilliseconds(50), + _ => Task.CompletedTask, + cancellationToken); + }, + (process, inProgressTests, _) => + { + dumped.Add(process); + annotations.Add(inProgressTests); + return Task.CompletedTask; + }, + CancellationToken.None); + + Assert.AreEqual(1, queryCount); + Assert.AreSequenceEqual(bottomUpTree, dumped); + + // Every dump is annotated with the answer from that one query, so no process triggers another. + Assert.HasCount(bottomUpTree.Length, annotations); + foreach ((string, int)[] annotation in annotations) + { + Assert.AreSame(annotations[0], annotation); + } + } + + [TestMethod] + public async Task QueryInProgressTestsWithTimeout_WhenTheReplyNeverArrives_ReturnsEmptyList() + { + // A connected-but-wedged host accepts the request and never replies, and the application token is + // not cancelled while the run is still in progress -- which is exactly when the deadline dump + // fires. So the bound inside the product is the only thing that can end this wait: the request + // here honors the token it is handed and nothing else, and never times out on its own. + Exception? loggedFailure = null; + + Task<(string, int)[]> query = HangDumpProcessLifetimeHandler.QueryInProgressTestsWithTimeoutAsync( + async queryCancellationToken => + { + await Task.Delay(Timeout.Infinite, queryCancellationToken); + return []; + }, + TimeSpan.FromMilliseconds(200), + ex => + { + loggedFailure = ex; + return Task.CompletedTask; + }, + CancellationToken.None); + + // Fail with a message rather than hanging the run if the bound is ever removed. + Task completed = await Task.WhenAny(query, Task.Delay(TimeSpan.FromSeconds(30), TestContext.CancellationToken)); + Assert.AreSame(query, completed, "The query did not give up on a reply that never arrives, so a wedged host would block the dump."); + + Assert.IsEmpty(await query); + + // The give-up is reported, so a missing in-progress-test list in a dump can be explained. + Assert.IsNotNull(loggedFailure); + } + + [TestMethod] + public async Task QueryInProgressTestsWithTimeout_WhenTheHostReplies_ReturnsTheAnswer() + { + // The bound must not get in the way of the healthy path, which answers in milliseconds. + (string, int)[] expected = [("Test1", 3), ("Test2", 7)]; + + (string, int)[] inProgressTests = await HangDumpProcessLifetimeHandler.QueryInProgressTestsWithTimeoutAsync( + _ => Task.FromResult(expected), + TimeSpan.FromSeconds(30), + _ => Task.CompletedTask, + CancellationToken.None); + + Assert.AreSequenceEqual(expected, inProgressTests); + } + + [TestMethod] + public async Task QueryInProgressTestsWithTimeout_WhenReportingTheFailureThrows_StillReturnsEmptyList() + { + // The empty list is what lets the dump go ahead after a failed query, and the delegate that reports + // the failure is a logger call -- logger providers can fail. If that throw escaped, a query failure + // would take the dump down with it, even though the query is explicitly best-effort. + (string, int)[] inProgressTests = await HangDumpProcessLifetimeHandler.QueryInProgressTestsWithTimeoutAsync( + _ => throw new InvalidOperationException("The consumer pipe is not connected."), + TimeSpan.FromSeconds(30), + _ => throw new InvalidOperationException("This logger provider is broken too."), + CancellationToken.None); + + Assert.IsEmpty(inProgressTests); + } + [TestMethod] [DataRow("Mini")] [DataRow("Heap")] diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Extensions/AbortAtDeadlineExtensionTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Extensions/AbortAtDeadlineExtensionTests.cs new file mode 100644 index 0000000000..7b451fe798 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Extensions/AbortAtDeadlineExtensionTests.cs @@ -0,0 +1,304 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Capabilities.TestFramework; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +using Moq; + +namespace Microsoft.Testing.Platform.UnitTests; + +#pragma warning disable TPEXP // IGracefulStopTestExecutionCapability is for evaluation purposes only. + +[TestClass] +public sealed class AbortAtDeadlineExtensionTests : IDisposable +{ + private static readonly DateTimeOffset Now = new(2030, 1, 1, 12, 0, 0, TimeSpan.Zero); + + private readonly CancellationTokenSource _cts = new(); + private readonly Mock _policiesService = new(); + private readonly Mock _capability = new(); + + public TestContext TestContext { get; set; } = null!; + + public void Dispose() => _cts.Dispose(); + + [TestMethod] + public async Task WhenGracefulStopFails_TheDeadlineVerdictIsReverted() + { + TaskCompletionSource reverted = new(TaskCreationOptions.RunContinuationsAsynchronously); + _policiesService.Setup(x => x.RevertDeadlineTriggered()).Callback(() => reverted.TrySetResult(true)); + _capability + .Setup(x => x.StopTestExecutionAsync(It.IsAny())) + .ThrowsAsync(new InvalidOperationException("This framework refuses to stop.")); + + using AbortAtDeadlineExtension extension = CreateExtension(deadlineIn: TimeSpan.Zero); + + // The stop was never accepted, so the run carries on and executes every test. Reporting exit code 15 + // for it would be a lie, so the verdict committed before the stop request has to be taken back. + await WaitForAsync(reverted.Task); + _policiesService.Verify(x => x.RevertDeadlineTriggered(), Times.Once); + } + + [TestMethod] + public async Task WhenGracefulStopSucceeds_TheDeadlineVerdictIsKept() + { + TaskCompletionSource stopped = new(TaskCreationOptions.RunContinuationsAsynchronously); + _capability + .Setup(x => x.StopTestExecutionAsync(It.IsAny())) + .Returns(() => + { + stopped.TrySetResult(true); + return Task.CompletedTask; + }); + + using AbortAtDeadlineExtension extension = CreateExtension(deadlineIn: TimeSpan.Zero); + + await WaitForAsync(stopped.Task); + _policiesService.Verify(x => x.ExecuteDeadlineCallbacksAsync(), Times.Once); + _policiesService.Verify(x => x.RevertDeadlineTriggered(), Times.Never); + } + + [TestMethod] + public async Task WhenTestExecutionCompleted_TheDeadlineDoesNotTrigger() + { + TaskCompletionSource triggered = new(TaskCreationOptions.RunContinuationsAsynchronously); + _policiesService.Setup(x => x.ExecuteDeadlineCallbacksAsync()).Callback(() => triggered.TrySetResult(true)).Returns(Task.CompletedTask); + + using AbortAtDeadlineExtension extension = CreateExtension(deadlineIn: TimeSpan.FromMilliseconds(300)); + + // The host signals this the moment the test framework invoker returns, which is well before the timer + // below fires. Everything after it -- draining the message bus, the reporters -- happens on a run that + // already executed every test, so a deadline reached during it must not truncate the verdict. + extension.NotifyTestExecutionCompleted(); + + Task completed = await Task.WhenAny(triggered.Task, Task.Delay(TimeSpan.FromSeconds(2), TestContext.CancellationToken)); + Assert.AreNotSame(triggered.Task, completed, "The deadline fired even though test execution had already completed."); + _policiesService.Verify(x => x.ExecuteDeadlineCallbacksAsync(), Times.Never); + _capability.Verify(x => x.StopTestExecutionAsync(It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task WhenPoliciesReportTestExecutionCompleted_TheDeadlineDoesNotTrigger() + { + TaskCompletionSource triggered = new(TaskCreationOptions.RunContinuationsAsynchronously); + _policiesService.SetupGet(x => x.IsTestExecutionCompleted).Returns(true); + _policiesService.Setup(x => x.ExecuteDeadlineCallbacksAsync()).Callback(() => triggered.TrySetResult(true)).Returns(Task.CompletedTask); + + using AbortAtDeadlineExtension extension = CreateExtension(deadlineIn: TimeSpan.Zero); + + Task completed = await Task.WhenAny(triggered.Task, Task.Delay(TimeSpan.FromSeconds(2), TestContext.CancellationToken)); + Assert.AreNotSame(triggered.Task, completed, "The deadline fired even though the shared policy state reported completed execution."); + _capability.Verify(x => x.StopTestExecutionAsync(It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task WhenTestExecutionIsStillRunning_TheDeadlineTriggers() + { + // Control for the test above: same timer, no completion signal, so the deadline must fire. Without + // this, that test would still pass if the timer were simply broken. + TaskCompletionSource triggered = new(TaskCreationOptions.RunContinuationsAsynchronously); + _policiesService.Setup(x => x.ExecuteDeadlineCallbacksAsync()).Callback(() => triggered.TrySetResult(true)).Returns(Task.CompletedTask); + + using AbortAtDeadlineExtension extension = CreateExtension(deadlineIn: TimeSpan.FromMilliseconds(300)); + + await WaitForAsync(triggered.Task); + _capability.Verify(x => x.StopTestExecutionAsync(It.IsAny()), Times.Once); + } + + [TestMethod] + public async Task WhenTestExecutionCompletesWhileTheDeadlineIsBeingReported_TheVerdictIsNotCommitted() + { + // The overlap the checks above cannot catch. The handler does not run on the timer callback: it yields + // and then reports, and the test framework invoker can return during exactly that window. Checking + // completion only before starting the handler would therefore still let the deadline commit its + // verdict and request a stop for a run in which every test had already finished (exit code 15 on a + // complete run). Gate the handler inside its reporting to make that window deterministic. + TaskCompletionSource reporting = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource abandoned = new(TaskCreationOptions.RunContinuationsAsynchronously); + + using AbortAtDeadlineExtension extension = CreateExtension( + deadlineIn: TimeSpan.Zero, + onLog: async logLevel => + { + switch (logLevel) + { + // "Deadline approaching ...", written before the verdict is committed. + case LogLevel.Information: + reporting.TrySetResult(true); + await release.Task; + break; + + // "... abandoning the graceful stop.", the only thing the handler does once it sees that + // test execution won the race. + case LogLevel.Debug: + abandoned.TrySetResult(true); + break; + } + }); + + // Park the handler mid-report, then let the invoker return underneath it. + await WaitForAsync(reporting.Task); + extension.NotifyTestExecutionCompleted(); + release.SetResult(true); + + await WaitForAsync(abandoned.Task); + _policiesService.Verify(x => x.ExecuteDeadlineCallbacksAsync(), Times.Never); + _policiesService.Verify(x => x.RevertDeadlineTriggered(), Times.Never); + _capability.Verify(x => x.StopTestExecutionAsync(It.IsAny()), Times.Never); + } + + [TestMethod] + public async Task WhenTestExecutionCompletesAfterTheDeadlineClaimedTheRun_TheVerdictStands() + { + // The mirror image of the test above, and the reason completion cannot simply revert the verdict: the + // graceful stop is what makes execution finish, so the completion that follows a claimed deadline is + // the stop taking effect, not a run that got there on its own. + TaskCompletionSource stopping = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + _capability + .Setup(x => x.StopTestExecutionAsync(It.IsAny())) + .Returns(async () => + { + stopping.TrySetResult(true); + await release.Task; + }); + + using AbortAtDeadlineExtension extension = CreateExtension(deadlineIn: TimeSpan.Zero); + + await WaitForAsync(stopping.Task); + extension.NotifyTestExecutionCompleted(); + release.SetResult(true); + + _policiesService.Verify(x => x.ExecuteDeadlineCallbacksAsync(), Times.Once); + _policiesService.Verify(x => x.RevertDeadlineTriggered(), Times.Never); + } + + [TestMethod] + public async Task TheVerdictAndTheGracefulStopBothPrecedeTheUserFacingMessage() + { + // Claiming the run closes the door on NotifyTestExecutionCompleted: once claimed, a completion can no + // longer disarm the deadline. From the claim until the stop is actually requested the run is therefore + // recorded as truncated while nothing has been asked to stop, and a run that finishes inside that + // window is ignored -- the stop then finds nothing left to stop, succeeds, and a run that executed + // every test still exits with code 15. The user-facing message is bounded by _reportTimeout, so + // sitting between the commit and the stop it alone could hold that window open for ten seconds. Pin + // both the commit and the stop ahead of it. + TaskCompletionSource displaying = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource release = new(TaskCreationOptions.RunContinuationsAsynchronously); + + using AbortAtDeadlineExtension extension = CreateExtension( + deadlineIn: TimeSpan.Zero, + onDisplay: async () => + { + displaying.TrySetResult(true); + await release.Task; + }); + + // Park the handler inside the message. Anything not done by now sits inside the window. + await WaitForAsync(displaying.Task); + _policiesService.Verify(x => x.ExecuteDeadlineCallbacksAsync(), Times.Once); + _capability.Verify(x => x.StopTestExecutionAsync(It.IsAny()), Times.Once); + + release.SetResult(true); + } + + [TestMethod] + public async Task WhenTheUserFacingMessageNeverCompletes_TheGracefulStopIsStillRequested() + { + // A wedged output device hands back a task that never completes, and a task that never completes never + // faults, so swallowing exceptions does not cover it -- only the bound does. The message is written + // after the stop, so what this pins is that the bound is still there: without it a wedged device would + // keep the handler task alive past the end of the run, and it would once again be able to swallow the + // stop entirely if the message ever moved back ahead of it. + TaskCompletionSource displaying = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource neverCompletes = new(TaskCreationOptions.RunContinuationsAsynchronously); + TaskCompletionSource stopped = new(TaskCreationOptions.RunContinuationsAsynchronously); + + _capability + .Setup(x => x.StopTestExecutionAsync(It.IsAny())) + .Returns(() => + { + stopped.TrySetResult(true); + return Task.CompletedTask; + }); + + using AbortAtDeadlineExtension extension = CreateExtension( + deadlineIn: TimeSpan.Zero, + onDisplay: () => + { + displaying.TrySetResult(true); + return neverCompletes.Task; + }, + reportTimeout: TimeSpan.FromMilliseconds(200)); + + await WaitForAsync(displaying.Task); + + // Already requested: the message runs after the stop, so a wedged device cannot swallow it. The bound + // is what then lets the handler finish rather than sitting on a task that never completes. + await WaitForAsync(stopped.Task); + _capability.Verify(x => x.StopTestExecutionAsync(It.IsAny()), Times.Once); + _policiesService.Verify(x => x.RevertDeadlineTriggered(), Times.Never); + + neverCompletes.SetResult(true); + } + + private async Task WaitForAsync(Task task) + { + Task completed = await Task.WhenAny(task, Task.Delay(TimeSpan.FromSeconds(30), TestContext.CancellationToken)); + Assert.AreSame(task, completed, "Timed out waiting for the deadline handler."); + await task; + } + + private AbortAtDeadlineExtension CreateExtension(TimeSpan deadlineIn, Func? onLog = null, Func? onDisplay = null, TimeSpan? reportTimeout = null) + { + Mock environment = new(); + _ = environment.Setup(x => x.GetEnvironmentVariable(It.IsAny())).Returns((string?)null); + _ = environment + .Setup(x => x.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE)) + .Returns((Now + deadlineIn).ToString("o", CultureInfo.InvariantCulture)); + + // A zero stop margin makes the stop instant the deadline itself, so deadlineIn is exactly how long the + // timer waits (measured against the fixed clock below, not wall-clock time when the test starts). + _ = environment + .Setup(x => x.GetEnvironmentVariable(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN)) + .Returns("0"); + + Mock clock = new(); + _ = clock.SetupGet(x => x.UtcNow).Returns(Now); + + Mock logger = new(); + _ = logger + .Setup(x => x.LogAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>())) + .Returns((LogLevel logLevel, string _, Exception? _, Func _) + => onLog is null ? Task.CompletedTask : onLog(logLevel)); + Mock loggerFactory = new(); + _ = loggerFactory.Setup(x => x.CreateLogger(It.IsAny())).Returns(logger.Object); + + Mock outputDevice = new(); + _ = outputDevice + .Setup(x => x.DisplayAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => onDisplay is null ? Task.CompletedTask : onDisplay()); + + Mock cancellationTokenSource = new(); + _ = cancellationTokenSource.SetupGet(x => x.CancellationToken).Returns(_cts.Token); + + return new AbortAtDeadlineExtension( + environment.Object, + clock.Object, + _capability.Object, + _policiesService.Object, + cancellationTokenSource.Object, + outputDevice.Object, + loggerFactory.Object, + reportTimeout); + } +} + +#pragma warning restore TPEXP diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Helpers/DeadlineHelperTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Helpers/DeadlineHelperTests.cs new file mode 100644 index 0000000000..771ce01aad --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Helpers/DeadlineHelperTests.cs @@ -0,0 +1,167 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Helpers; + +using Moq; + +namespace Microsoft.Testing.Platform.UnitTests; + +[TestClass] +public sealed class DeadlineHelperTests +{ + [TestMethod] + [DataRow(null)] + [DataRow("")] + [DataRow(" ")] + [DataRow("not-a-date")] + [DataRow("12345")] + [DataRow("2030-13-01T00:00:00Z")] // invalid month + public void TryGetDeadline_WhenUnsetOrMalformed_ReturnsFalse(string? raw) + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE, raw); + + bool result = DeadlineHelper.TryGetDeadline(environment, out DateTimeOffset deadlineUtc); + + Assert.IsFalse(result); + Assert.AreEqual(default, deadlineUtc); + } + + [TestMethod] + public void TryGetDeadline_WhenUtcInstant_ReturnsInstantInUtc() + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE, "2030-01-01T00:00:00Z"); + + bool result = DeadlineHelper.TryGetDeadline(environment, out DateTimeOffset deadlineUtc); + + Assert.IsTrue(result); + Assert.AreEqual(new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero), deadlineUtc); + Assert.AreEqual(TimeSpan.Zero, deadlineUtc.Offset); + } + + [TestMethod] + public void TryGetDeadline_WhenInstantHasOffset_ConvertsToUtc() + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE, "2030-01-01T00:00:00+02:00"); + + bool result = DeadlineHelper.TryGetDeadline(environment, out DateTimeOffset deadlineUtc); + + Assert.IsTrue(result); + // 00:00 at +02:00 is 22:00 the previous day in UTC. + Assert.AreEqual(new DateTimeOffset(2029, 12, 31, 22, 0, 0, TimeSpan.Zero), deadlineUtc); + Assert.AreEqual(TimeSpan.Zero, deadlineUtc.Offset); + } + + [TestMethod] + public void TryGetDeadline_WhenInstantHasNoOffset_IsAssumedUtc() + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE, "2030-01-01T00:00:00"); + + bool result = DeadlineHelper.TryGetDeadline(environment, out DateTimeOffset deadlineUtc); + + Assert.IsTrue(result); + Assert.AreEqual(new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero), deadlineUtc); + } + + [TestMethod] + [DataRow("45", 45)] + [DataRow("45s", 45)] + [DataRow("2m", 120)] + [DataRow("0", 0)] + public void GetStopMargin_WhenParsable_ReturnsParsedValue(string raw, int expectedSeconds) + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN, raw); + + Assert.AreEqual(TimeSpan.FromSeconds(expectedSeconds), DeadlineHelper.GetStopMargin(environment)); + } + + [TestMethod] + [DataRow(null)] + [DataRow("")] + [DataRow(" ")] + [DataRow("abc")] + [DataRow("-1s")] // negative is not accepted by the parser, so the default is used + public void GetStopMargin_WhenUnsetOrUnparsable_ReturnsDefault(string? raw) + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_STOP_MARGIN, raw); + + Assert.AreEqual(TimeSpan.FromSeconds(60), DeadlineHelper.GetStopMargin(environment)); + } + + [TestMethod] + [DataRow("15", 15)] + [DataRow("15s", 15)] + [DataRow("1m", 60)] + public void GetDumpMargin_WhenParsable_ReturnsParsedValue(string raw, int expectedSeconds) + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN, raw); + + Assert.AreEqual(TimeSpan.FromSeconds(expectedSeconds), DeadlineHelper.GetDumpMargin(environment)); + } + + [TestMethod] + [DataRow(null)] + [DataRow("")] + [DataRow("nonsense")] + public void GetDumpMargin_WhenUnsetOrUnparsable_ReturnsDefault(string? raw) + { + IEnvironment environment = CreateEnvironment(EnvironmentVariableConstants.TESTINGPLATFORM_DEADLINE_DUMP_MARGIN, raw); + + Assert.AreEqual(TimeSpan.FromSeconds(30), DeadlineHelper.GetDumpMargin(environment)); + } + + [TestMethod] + public void SubtractSaturating_WhenNoUnderflow_SubtractsMargin() + { + var instant = new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero); + + DateTimeOffset result = DeadlineHelper.SubtractSaturating(instant, TimeSpan.FromSeconds(60)); + + Assert.AreEqual(new DateTimeOffset(2029, 12, 31, 23, 59, 0, TimeSpan.Zero), result); + } + + [TestMethod] + public void SubtractSaturating_WhenMarginIsZero_ReturnsInstant() + { + var instant = new DateTimeOffset(2030, 1, 1, 0, 0, 0, TimeSpan.Zero); + + Assert.AreEqual(instant, DeadlineHelper.SubtractSaturating(instant, TimeSpan.Zero)); + } + + [TestMethod] + public void SubtractSaturating_WhenMarginWouldUnderflow_ClampsToMinValue() + { + DateTimeOffset instant = DateTimeOffset.MinValue.AddSeconds(10); + + DateTimeOffset result = DeadlineHelper.SubtractSaturating(instant, TimeSpan.FromSeconds(60)); + + Assert.AreEqual(DateTimeOffset.MinValue, result); + } + + [TestMethod] + public void SubtractSaturating_WhenMarginEqualsAvailableRange_ReturnsMinValue() + { + DateTimeOffset instant = DateTimeOffset.MinValue.AddSeconds(60); + + // margin (60s) is not greater than the available range (60s), so the exact subtraction is used + // and lands precisely on MinValue. + DateTimeOffset result = DeadlineHelper.SubtractSaturating(instant, TimeSpan.FromSeconds(60)); + + Assert.AreEqual(DateTimeOffset.MinValue, result); + } + + private static IEnvironment CreateEnvironment(string variableName, string? value) + { + Mock environment = new(); + _ = environment.Setup(x => x.GetEnvironmentVariable(It.IsAny())).Returns((string?)null); + + // A null value models "variable unset"; the default mock already returns null, so only wire up + // an explicit (possibly empty/whitespace) value. + if (value is not null) + { + _ = environment.Setup(x => x.GetEnvironmentVariable(variableName)).Returns(value); + } + + return environment.Object; + } +} diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Hosts/CommonHostTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Hosts/CommonHostTests.cs index 2af9bb4e36..afa6f4187c 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Hosts/CommonHostTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Hosts/CommonHostTests.cs @@ -50,9 +50,12 @@ public async Task ExecuteRequestAsync_WhenSessionIsCancelled_UsesCancellationTok .Setup(x => x.ExecuteAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ThrowsAsync(new OperationCanceledException(cancellationToken)); + Mock policiesServiceMock = new(); + ServiceProvider serviceProvider = new(); serviceProvider.AddService(testFrameworkInvokerMock.Object); serviceProvider.AddService(new TestCoverageResult()); + serviceProvider.AddService(policiesServiceMock.Object); Mock baseMessageBusMock = new(); baseMessageBusMock.Setup(x => x.DrainDataAsync()).Returns(Task.CompletedTask); @@ -73,6 +76,11 @@ await TestableCommonHost.ExecuteRequestForTestingAsync( outputDeviceMock.Verify(x => x.DisplayAfterSessionEndRunAsync(It.IsAny()), Times.Once); testSessionLifetimeHandlerMock.Verify(x => x.OnTestSessionFinishingAsync(It.IsAny()), Times.Once); + + // Disarming happens in a finally around the invoker, so it must also happen when the invoker threw + // because the session was canceled. Otherwise a deadline reached while the reporters finalize an + // already-canceled run would still mark it as truncated. + policiesServiceMock.Verify(x => x.NotifyTestExecutionCompleted(), Times.Once); } [TestMethod] @@ -116,6 +124,7 @@ public async Task ExecuteRequestAsync_WhenSessionIsCancelled_DisablesTheMessageB ServiceProvider serviceProvider = new(); serviceProvider.AddService(testFrameworkInvokerMock.Object); serviceProvider.AddService(new TestCoverageResult()); + serviceProvider.AddService(new Mock().Object); Mock baseMessageBusMock = new(); baseMessageBusMock.Setup(x => x.DrainDataAsync()).Returns(Task.CompletedTask); @@ -154,9 +163,12 @@ public async Task ExecuteRequestAsync_WhenDisablingTheMessageBusFails_DoesNotMas .Setup(x => x.ExecuteAsync(It.IsAny(), It.IsAny(), It.IsAny())) .ThrowsAsync(new InvalidOperationException("test framework failure")); + Mock policiesServiceMock = new(); + ServiceProvider serviceProvider = new(); serviceProvider.AddService(testFrameworkInvokerMock.Object); serviceProvider.AddService(new TestCoverageResult()); + serviceProvider.AddService(policiesServiceMock.Object); Mock baseMessageBusMock = new(); baseMessageBusMock.Setup(x => x.DrainDataAsync()).Returns(Task.CompletedTask); @@ -174,6 +186,10 @@ public async Task ExecuteRequestAsync_WhenDisablingTheMessageBusFails_DoesNotMas // The safety net is best effort: it must never replace the exception that is already propagating. Assert.AreEqual("test framework failure", ex.Message); baseMessageBusMock.Verify(x => x.DisableAsync(), Times.Once); + + // Disarming sits in a finally around the invoker, so a failing test framework must not leave the + // deadline armed while the session tears down. + policiesServiceMock.Verify(x => x.NotifyTestExecutionCompleted(), Times.Once); } [TestMethod] diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj index b82c156f88..fecb82211b 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Microsoft.Testing.Platform.UnitTests.csproj @@ -36,6 +36,7 @@ + diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/StopPoliciesServiceTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/StopPoliciesServiceTests.cs index ebc23ee8c6..013e07a51a 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/StopPoliciesServiceTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/Services/StopPoliciesServiceTests.cs @@ -36,6 +36,248 @@ public void IsAbortTriggered_InitiallyFalse() Assert.IsFalse(service.IsAbortTriggered); } + [TestMethod] + public void IsTestExecutionCompleted_InitiallyFalse() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + Assert.IsFalse(service.IsTestExecutionCompleted); + } + + [TestMethod] + public void NotifyTestExecutionCompleted_SetsIsTestExecutionCompleted() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + + service.NotifyTestExecutionCompleted(); + + Assert.IsTrue(service.IsTestExecutionCompleted); + } + + [TestMethod] + public void NotifyTestExecutionCompleted_DoesNotAffectDeadlineTriggered() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + + service.NotifyTestExecutionCompleted(); + + // Completing execution gates future deadlines; it must not itself look like a deadline truncation. + Assert.IsFalse(service.IsDeadlineTriggered); + } + + [TestMethod] + public async Task RevertDeadlineTriggered_ClearsIsDeadlineTriggered() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + + await service.ExecuteDeadlineCallbacksAsync(); + Assert.IsTrue(service.IsDeadlineTriggered); + + // The deadline outcome is recorded before the graceful stop is requested, so it has to be + // retractable when that request is rejected and nothing was actually truncated. + service.RevertDeadlineTriggered(); + + Assert.IsFalse(service.IsDeadlineTriggered); + } + + [TestMethod] + public void RevertDeadlineTriggered_WhenNeverTriggered_LeavesFlagFalse() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + + service.RevertDeadlineTriggered(); + + Assert.IsFalse(service.IsDeadlineTriggered); + } + + [TestMethod] + public void IsDeadlineTriggered_InitiallyFalse() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + Assert.IsFalse(service.IsDeadlineTriggered); + } + + [TestMethod] + public async Task ExecuteDeadlineCallbacksAsync_SetsIsDeadlineTriggered() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + + await service.ExecuteDeadlineCallbacksAsync(); + + Assert.IsTrue(service.IsDeadlineTriggered); + } + + [TestMethod] + public async Task ExecuteDeadlineCallbacksAsync_InvokesRegisteredCallback() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + + int invocationCount = 0; + await service.RegisterOnDeadlineCallbackAsync(() => + { + invocationCount++; + return Task.CompletedTask; + }); + + await service.ExecuteDeadlineCallbacksAsync(); + + Assert.AreEqual(1, invocationCount); + } + + [TestMethod] + public async Task ExecuteDeadlineCallbacksAsync_IsOneShot() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + + int invocationCount = 0; + await service.RegisterOnDeadlineCallbackAsync(() => + { + invocationCount++; + return Task.CompletedTask; + }); + + await service.ExecuteDeadlineCallbacksAsync(); + await service.ExecuteDeadlineCallbacksAsync(); + + Assert.AreEqual(1, invocationCount); + } + + [TestMethod] + public async Task RegisterOnDeadlineCallbackAsync_InvokesCallbackExactlyOnceIfAlreadyTriggered() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + await service.ExecuteDeadlineCallbacksAsync(); + + int invocationCount = 0; + await service.RegisterOnDeadlineCallbackAsync(() => + { + invocationCount++; + return Task.CompletedTask; + }); + + // The deadline is one-shot, so registering after it fired must invoke the callback right away and + // must not leave it queued for a second, never-arriving trigger. + await service.ExecuteDeadlineCallbacksAsync(); + + Assert.AreEqual(1, invocationCount); + } + + [TestMethod] + public async Task RegisterOnDeadlineCallbackAsync_RacingTheTrigger_InvokesEveryCallbackExactlyOnce() + { + // Registration used to be able to lose a callback: the registering thread could read the trigger flag + // as false, the trigger could then commit and snapshot a still-empty queue, and only afterwards would + // the callback be enqueued -- where a one-shot deadline never reaches it. Hammer registration against + // the trigger and assert every callback ran exactly once. + for (int attempt = 0; attempt < 50; attempt++) + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + int[] invocationCounts = new int[8]; + + // Async start gate so every task is released at the same moment. Barrier would be the obvious + // choice but it is unsupported on browser, which this project targets. + TaskCompletionSource start = new(TaskCreationOptions.RunContinuationsAsynchronously); + + var tasks = new List(); + for (int i = 0; i < invocationCounts.Length; i++) + { + int index = i; + tasks.Add(Task.Run( + async () => + { + await start.Task; + await service.RegisterOnDeadlineCallbackAsync(() => + { + Interlocked.Increment(ref invocationCounts[index]); + return Task.CompletedTask; + }); + }, + TestContext.CancellationToken)); + } + + tasks.Add(Task.Run( + async () => + { + await start.Task; + await service.ExecuteDeadlineCallbacksAsync(); + }, + TestContext.CancellationToken)); + + start.SetResult(true); + await Task.WhenAll(tasks); + + for (int i = 0; i < invocationCounts.Length; i++) + { + Assert.AreEqual(1, invocationCounts[i], $"Callback {i} was invoked {invocationCounts[i]} times on attempt {attempt}."); + } + } + } + + [TestMethod] + public async Task RevertDeadlineTriggered_ClearsTheVerdictAndKeepsCallbacksOneShot() + { + StopPoliciesService service = new(_cancellationTokenSource.Object); + await service.ExecuteDeadlineCallbacksAsync(); + Assert.IsTrue(service.IsDeadlineTriggered); + + service.RevertDeadlineTriggered(); + + Assert.IsFalse(service.IsDeadlineTriggered); + } + + [TestMethod] + public async Task RevertDeadlineTriggered_KeepsCallbacksOneShot() + { + // Reverting undoes the exit-code verdict only. If it also re-armed the callback gate, the next + // trigger would run every callback a second time. + StopPoliciesService service = new(_cancellationTokenSource.Object); + + int invocationCount = 0; + await service.RegisterOnDeadlineCallbackAsync(() => + { + invocationCount++; + return Task.CompletedTask; + }); + + await service.ExecuteDeadlineCallbacksAsync(); + service.RevertDeadlineTriggered(); + await service.ExecuteDeadlineCallbacksAsync(); + + Assert.AreEqual(1, invocationCount); + } + + [TestMethod] + public async Task RegisterOnDeadlineCallbackAsync_AfterRevert_InvokesCallbackImmediately() + { + // The callbacks have already run, so a registration arriving after the revert has missed the only + // trigger there will ever be. Queueing it instead would drop it: nothing drains the list any more. + StopPoliciesService service = new(_cancellationTokenSource.Object); + await service.ExecuteDeadlineCallbacksAsync(); + service.RevertDeadlineTriggered(); + + int invocationCount = 0; + await service.RegisterOnDeadlineCallbackAsync(() => + { + invocationCount++; + return Task.CompletedTask; + }); + + Assert.AreEqual(1, invocationCount); + } + + [TestMethod] + public async Task RevertDeadlineTriggered_DoesNotResurrectTheVerdict() + { + // A second trigger after a revert must stay a no-op: the run was not truncated, so it must not be + // reported as stopped at the deadline. + StopPoliciesService service = new(_cancellationTokenSource.Object); + await service.ExecuteDeadlineCallbacksAsync(); + service.RevertDeadlineTriggered(); + + await service.ExecuteDeadlineCallbacksAsync(); + + Assert.IsFalse(service.IsDeadlineTriggered); + } + [TestMethod] public async Task ExecuteMaxFailedTestsCallbacksAsync_SetsIsMaxFailedTestsTriggered() {