diff --git a/.github/scripts/pr-validation.Tests.ps1 b/.github/scripts/pr-validation.Tests.ps1 new file mode 100644 index 00000000000..7418d9c9304 --- /dev/null +++ b/.github/scripts/pr-validation.Tests.ps1 @@ -0,0 +1,188 @@ +$ErrorActionPreference = 'Stop' +$root = Join-Path $PSScriptRoot ('.pr-validation-tests-' + [Guid]::NewGuid()) +$count = 0 +New-Item -ItemType Directory $root | Out-Null + +function Assert($Condition, $Message) { + if (!$Condition) { throw $Message } +} + +function Assert-Snapshot($Request) { + Assert ($Request.resources.repositories.self.refName -ceq 'refs/pull/42/head') 'Wrong YAML ref' + Assert ($Request.resources.repositories.self.version -ceq ('a' * 40)) 'Wrong YAML revision' + Assert ($Request.templateParameters.prNumber -is [string] -and $Request.templateParameters.prNumber -ceq '42') 'PR number must be a string' + Assert ($Request.templateParameters.headSha -ceq ('a' * 40)) 'Wrong head snapshot' + Assert ($Request.templateParameters.baseSha -ceq ('b' * 40)) 'Wrong base snapshot' +} + +function Invoke-WebRequest { + [CmdletBinding()] + param($Uri, $Method, $Headers, $UserAgent, $ContentType, $MaximumRedirection, [switch]$SkipHttpErrorCheck, $Body) + $calls.Add($PSBoundParameters) + Assert ($responses.ContainsKey($Uri)) "Unexpected URL: $Uri" + Assert ($Headers.Authorization -ceq "Bearer $expectedToken") 'Wrong bearer token' + Assert ($Headers.Accept -ceq 'application/json' -and $ContentType -ceq 'application/json') 'Wrong JSON headers' + Assert ($UserAgent -ceq 'fsharp-pr-validation') 'Missing user agent' + Assert ($MaximumRedirection -ceq 0 -and $SkipHttpErrorCheck -and $PSBoundParameters.ErrorAction -eq 'Stop') 'HTTP handling is not fail-closed' + $response = $responses[$Uri] + Assert ($Method -ceq $response.Method) 'Wrong HTTP method' + if ($response.Failure) { throw 'Network failed (test-token)' } + @{ + StatusCode = $response.Status + Content = if ($response.ContainsKey('Raw')) { $response.Raw } else { ConvertTo-Json -InputObject $response.Body -Depth 10 -Compress } + } +} + +function Test-Case($Name, [scriptblock]$Change = {}, $ExpectedError = '', $Mode = 'Prepare', $CallCount = -1) { + $event = @{ + comment = @{ body = "/dart $('a' * 40)"; user = @{ login = 'maintainer'; type = 'User' } } + issue = @{ number = 42; pull_request = @{} } + } + $pr = @{ + state = 'open' + head = @{ sha = 'a' * 40; repo = @{ full_name = 'contributor/fsharp' } } + base = @{ sha = 'b' * 40; ref = 'main'; repo = @{ full_name = 'dotnet/fsharp' } } + } + $permission = @{ permission = 'write' } + $request = @{ + resources = @{ repositories = @{ self = @{ refName = 'refs/pull/42/head'; version = 'a' * 40 } } } + templateParameters = @{ prNumber = '42'; headSha = 'a' * 40; baseSha = 'b' * 40 } + } + $permissionUri = 'https://api.github.com/repos/dotnet/fsharp/collaborators/maintainer/permission' + $pullUri = 'https://api.github.com/repos/dotnet/fsharp/pulls/42' + $memberUri = 'https://api.github.com/orgs/microsoft/members/maintainer' + $queueUri = 'https://dev.azure.com/devdiv/DevDiv/_apis/pipelines/123/runs?api-version=7.1' + $reportUri = 'https://api.github.com/repos/dotnet/fsharp/issues/42/comments' + $responses = @{ + $permissionUri = @{ Method = 'GET'; Status = 200; Body = $permission } + $pullUri = @{ Method = 'GET'; Status = 200; Body = $pr } + $memberUri = @{ Method = 'GET'; Status = 204 } + $queueUri = @{ Method = 'POST'; Status = 200; Body = @{ id = 456 } } + $reportUri = @{ Method = 'POST'; Status = 201; Body = @{ id = 789 } } + } + $calls = [Collections.Generic.List[object]]::new() + $env:GITHUB_EVENT_PATH = Join-Path $root 'event.json' + $env:GITHUB_OUTPUT = Join-Path $root 'output' + $env:GITHUB_REPOSITORY = 'dotnet/fsharp'; $env:GITHUB_ACTOR = 'rerunning-admin' + $env:GITHUB_SERVER_URL = 'https://github.com'; $env:GITHUB_RUN_ID = '987' + $env:GITHUB_TOKEN = if ($Mode -eq 'Membership') { 'membership-token' } else { 'github-token' } + $env:PIPELINE_ID = '123'; $env:AZDO_TOKEN = 'test-token' + $env:REQUEST_BODY = ConvertTo-Json -InputObject $request -Depth 10 -Compress + $env:RUN_URL = ''; $env:HEAD_SHA = 'a' * 40; $env:BASE_SHA = 'b' * 40 + $expectedToken = if ($Mode -eq 'Queue') { $env:AZDO_TOKEN } else { $env:GITHUB_TOKEN } + . $Change + ConvertTo-Json -InputObject $event -Depth 10 | Set-Content -LiteralPath $env:GITHUB_EVENT_PATH -Encoding utf8 + Remove-Item -LiteralPath $env:GITHUB_OUTPUT -ErrorAction SilentlyContinue + $failure = $null + try { $stdout = @(& "$PSScriptRoot\pr-validation.ps1" -Mode $Mode) } + catch { $failure = "$_" } + if ($ExpectedError) { + Assert ($failure -and $failure -match $ExpectedError) "$Name failed for wrong reason: $failure" + Assert (!(Test-Path -LiteralPath $env:GITHUB_OUTPUT)) "$Name wrote outputs on failure" + Assert ($failure -notmatch 'test-token|github-token|membership-token') 'Leaked token' + } + else { + Assert (!$failure) "$Name failed: $failure" + Assert ($stdout.Count -eq 0) "$Name leaked stdout" + switch ($Mode) { + 'Prepare' { + $lines = @(Get-Content -LiteralPath $env:GITHUB_OUTPUT) + Assert ($lines.Count -eq 3 -and $lines[1] -ceq "head=$('a' * 40)" -and $lines[2] -ceq "base=$('b' * 40)") 'Wrong snapshot outputs' + Assert-Snapshot (ConvertFrom-Json $lines[0].Substring(5)) + Assert ($calls[0].Uri -ceq $permissionUri -and $calls[1].Uri -ceq $pullUri) 'Wrong requester or PR' + } + 'Queue' { + Assert-Snapshot (ConvertFrom-Json $calls[0].Body) + Assert ((Get-Content -LiteralPath $env:GITHUB_OUTPUT -Raw).Trim() -ceq 'url=https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=456') 'Wrong run link' + } + 'Report' { + $body = (ConvertFrom-Json $calls[0].Body).body + $expected = if ($env:RUN_URL) { '[F# Apex run](https://example.invalid/run) requested by @maintainer.' } + else { 'F# Apex request failed. [Workflow details](https://github.com/dotnet/fsharp/actions/runs/987).' } + Assert ($body.StartsWith($expected)) 'Wrong PR feedback' + Assert ($body.Contains('`/dart `') -and $body.Contains('`/pr-val `')) 'Missing retry guidance' + if ($env:RUN_URL) { Assert ($body.Contains("Reviewed PR head and pipeline YAML: ``$env:HEAD_SHA``") -and $body.Contains("Base: ``$env:BASE_SHA``")) 'Missing snapshot identifiers' } + } + } + if ($CallCount -lt 0) { $CallCount = if ($Mode -eq 'Prepare') { 2 } else { 1 } } + } + if ($CallCount -ge 0) { Assert ($calls.Count -eq $CallCount) "$Name made unexpected HTTP calls" } + $script:count++ + Write-Host "PASS: $Name" +} + +try { + foreach ($command in "/dart $('a' * 40)", "/pr-val $('a' * 40)", " `n/pr-val $('a' * 40) `n") { + foreach ($authorRepo in 'dotnet/fsharp', 'contributor/fsharp') { + Test-Case "$command from $authorRepo" { $event.comment.body = $command; $pr.head.repo.full_name = $authorRepo } + } + } + foreach ($command in '/dart', '/pr-val', '/dart abc1234', 'please /dart', '`/dart`', "/dart`n/pr-val", '/dart-extra', '/DART', "/dart`nextra", "/dart $('A' * 40)", "/dart $('a' * 39)", "/dart $('a' * 41)", "/dart $('g' * 40)", "/dart $('a' * 40) extra", "/dart`n$('a' * 40)", "/dart $('a' * 40)", '', $null, 42) { + Test-Case "reject command '$command'" { $event.comment.body = $command } 'standalone' -CallCount 0 + } + foreach ($alias in 'dart', 'pr-val') { + Test-Case "$alias rejects base SHA" { $event.comment.body = "/$alias $('b' * 40)" } 'current PR head' -CallCount 2 + Test-Case "$alias rejects stale reviewed SHA" { $event.comment.body = "/$alias $('c' * 40)" } 'current PR head' -CallCount 2 + Test-Case "$alias rejects updated PR head" { $event.comment.body = "/$alias $('a' * 40)"; $pr.head.sha = 'c' * 40 } 'current PR head' -CallCount 2 + } + foreach ($access in 'write', 'maintain', 'admin') { Test-Case "permission $access" { $permission.permission = $access } } + foreach ($access in 'read', 'triage', 'WRITE', '', $null, @('write')) { Test-Case "reject permission '$access'" { $permission.permission = $access } 'write access' -CallCount 1 } + Test-Case 'bot' { $event.comment.user.type = 'Bot' } 'user comments' -CallCount 0 + Test-Case 'array user type' { $event.comment.user.type = @('User') } 'user comments' -CallCount 0 + Test-Case 'issue' { $event.issue.Remove('pull_request') } 'dotnet/fsharp PRs' -CallCount 0 + Test-Case 'false PR marker' { $event.issue.pull_request = $false } 'dotnet/fsharp PRs' -CallCount 0 + Test-Case 'array event' { $event = @($event) } 'dotnet/fsharp PRs' -CallCount 0 + Test-Case 'array permission response' { $responses[$permissionUri].Body = @($permission) } 'write access' + Test-Case 'array PR response' { $responses[$pullUri].Body = @($pr) } 'open PRs' + foreach ($repo in 'other/fsharp', 'dotnet/other', 'DotNet/fsharp') { + Test-Case "wrong repo $repo" { $env:GITHUB_REPOSITORY = $repo } 'dotnet/fsharp PRs' -CallCount 0 + } + foreach ($id in '1; command', '42', $null, 0, -1, 1.5, $true, 9007199254740992) { + Test-Case "bad PR number '$id'" { $event.issue.number = $id } 'PR number' -CallCount 0 + } + Test-Case 'closed PR' { $pr.state = 'closed' } 'open PRs' + Test-Case 'uppercase state' { $pr.state = 'OPEN' } 'open PRs' + Test-Case 'array state' { $pr.state = @('open') } 'open PRs' + Test-Case 'array target' { $pr.base.repo.full_name = @('dotnet/fsharp') } 'main' + Test-Case 'array branch' { $pr.base.ref = @('main') } 'main' + foreach ($branch in 'release/test', 'Main') { Test-Case "non-main $branch" { $pr.base.ref = $branch } 'main' } + Test-Case 'foreign target' { $pr.base.repo.full_name = 'other/fsharp' } 'main' + foreach ($side in 'head', 'base') { + foreach ($sha in 'abcdef0', $null, ('A' * 40), (('a' * 40) + "`n"), 123, @('a' * 40)) { + Test-Case "bad $side SHA '$sha'" { $pr[$side].sha = $sha } 'full head/base' + } + } + Test-Case 'membership original commenter' -Mode Membership + foreach ($status in 200, 201, 301, 302, 307, 308, 401, 403, 404, 429, 500) { + Test-Case "membership HTTP $status" { $responses[$memberUri].Status = $status } "HTTP $status" Membership + } + foreach ($endpoint in 'permissionUri', 'pullUri', 'memberUri', 'queueUri', 'reportUri') { + $mode = switch ($endpoint) { memberUri { 'Membership' }; queueUri { 'Queue' }; reportUri { 'Report' }; default { 'Prepare' } } + Test-Case "$endpoint network failure" { $responses[(Get-Variable $endpoint -ValueOnly)].Failure = $true } 'HTTP request failed' $mode + Test-Case "$endpoint API failure" { $responses[(Get-Variable $endpoint -ValueOnly)].Status = 500 } 'HTTP 500' $mode + } + Test-Case 'queue snapshot' -Mode Queue + Test-Case 'queue created' { $responses[$queueUri].Status = 201 } -Mode Queue + foreach ($json in '{"id":456.0}', '{"id":4.56e2}') { + Test-Case "integral JSON number $json" { $responses[$queueUri].Raw = $json } -Mode Queue + } + foreach ($pipeline in '', '0', '123/path', "123`n", ' 123', '+123') { + Test-Case "invalid pipeline '$pipeline'" { $env:PIPELINE_ID = $pipeline } 'Configure FSHARP_APEX_PIPELINE_ID' Queue 0 + } + foreach ($token in '', ' ') { Test-Case 'missing queue token' { $env:AZDO_TOKEN = $token } 'access token' Queue 0 } + Test-Case 'invalid request JSON' { $env:REQUEST_BODY = '{' } 'JSON' Queue 0 + foreach ($status in 301, 302, 303, 307, 308, 401, 403, 429, 500) { + Test-Case "queue HTTP $status" { $responses[$queueUri].Status = $status } "HTTP $status" Queue + } + foreach ($id in $null, 0, -1, 1.5, '456', $true, 9007199254740992, @(456)) { + Test-Case "invalid run ID '$id'" { $responses[$queueUri].Body.id = $id } 'valid run ID' Queue + } + foreach ($json in '{}', 'null', '[{"id":456}]', '{"id":[456]}', '{"id":{}}', '{"ID":456}') { + Test-Case "invalid run $json" { $responses[$queueUri].Raw = $json } 'valid run ID' Queue + } + Test-Case 'invalid response JSON' { $responses[$queueUri].Raw = '' } 'JSON' Queue + Test-Case 'report success' { $env:RUN_URL = 'https://example.invalid/run' } -Mode Report + Test-Case 'report failure' -Mode Report + Write-Host "$count tests passed." +} +finally { Remove-Item -LiteralPath $root -Recurse -Force } diff --git a/.github/scripts/pr-validation.ps1 b/.github/scripts/pr-validation.ps1 new file mode 100644 index 00000000000..14fe7270aa8 --- /dev/null +++ b/.github/scripts/pr-validation.ps1 @@ -0,0 +1,89 @@ +param([Parameter(Mandatory)][ValidateSet('Prepare', 'Membership', 'Queue', 'Report')][string]$Mode) + +$ErrorActionPreference = 'Stop' + +function Test-SafeId($Value) { + ($Value -is [long] -or $Value -is [int] -or $Value -is [double]) -and + $Value -gt 0 -and $Value -le 9007199254740991 -and [Math]::Truncate($Value) -eq $Value +} + +function Invoke-Api($Uri, $Token, $Method = 'GET', $Body = $null, $ExpectedStatus = 200) { + $options = @{ + Uri = $Uri; Method = $Method; Headers = @{ Authorization = "Bearer $Token"; Accept = 'application/json' } + UserAgent = 'fsharp-pr-validation'; ContentType = 'application/json' + MaximumRedirection = 0; SkipHttpErrorCheck = $true; ErrorAction = 'Stop' + } + if ($null -ne $Body) { $options.Body = ConvertTo-Json -InputObject $Body -Depth 10 -Compress } + try { $response = Invoke-WebRequest @options } + catch { throw "HTTP request failed for $Uri." } + if ($response.StatusCode -notin $ExpectedStatus) { throw "HTTP $($response.StatusCode) from $Uri." } + if ($response.StatusCode -ne 204) { ConvertFrom-Json -InputObject $response.Content -NoEnumerate } +} + +if ($Mode -ne 'Queue') { + $event = Get-Content -LiteralPath $env:GITHUB_EVENT_PATH -Raw | ConvertFrom-Json -NoEnumerate + $prNumber = $event.issue.number + if ($env:GITHUB_REPOSITORY -cne 'dotnet/fsharp' -or $event -is [array] -or + $event.issue.pull_request -isnot [System.Management.Automation.PSCustomObject] -or !(Test-SafeId $prNumber)) { + throw 'Only dotnet/fsharp PRs with a valid PR number are supported.' + } + # Always use the original commenter, not the actor rerunning the workflow. + $login = [Uri]::EscapeDataString($event.comment.user.login) + $repoApi = 'https://api.github.com/repos/dotnet/fsharp' +} + +switch ($Mode) { + 'Prepare' { + if ($event.comment.user.type -isnot [string] -or $event.comment.user.type -cne 'User') { throw 'Only user comments can request validation.' } + if ($event.comment.body -isnot [string] -or $event.comment.body.Trim() -cnotmatch '\A/(dart|pr-val) ([0-9a-f]{40})\z') { + throw 'Use a standalone /dart or /pr-val with the full 40-character current PR head SHA.' + } + $requestedSha = $Matches[2] + $permission = Invoke-Api "$repoApi/collaborators/$login/permission" $env:GITHUB_TOKEN + if ($permission -is [array] -or $permission.permission -isnot [string] -or + $permission.permission -cnotin 'write', 'maintain', 'admin') { throw 'The requester needs repository write access.' } + $pr = Invoke-Api "$repoApi/pulls/$prNumber" $env:GITHUB_TOKEN + if ($pr -is [array] -or $pr.state -isnot [string] -or $pr.state -cne 'open' -or + $pr.base.repo.full_name -isnot [string] -or $pr.base.repo.full_name -cne 'dotnet/fsharp' -or + $pr.base.ref -isnot [string] -or $pr.base.ref -cne 'main') { + throw 'Only open PRs targeting dotnet/fsharp main are supported.' + } + $headSha, $baseSha = $pr.head.sha, $pr.base.sha + foreach ($sha in $headSha, $baseSha) { + if ($sha -isnot [string] -or $sha -cnotmatch '\A[0-9a-f]{40}\z') { throw 'GitHub did not return full head/base commit SHAs.' } + } + if ($requestedSha -cne $headSha) { + throw 'The requested SHA does not match the current PR head. Review the current commit and request a new run.' + } + $request = @{ + resources = @{ repositories = @{ self = @{ refName = "refs/pull/$prNumber/head"; version = $requestedSha } } } + templateParameters = @{ prNumber = "$prNumber"; headSha = $headSha; baseSha = $baseSha } + } + $json = ConvertTo-Json -InputObject $request -Depth 10 -Compress + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Encoding utf8 -Value "body=$json", "head=$headSha", "base=$baseSha" + } + 'Membership' { + $null = Invoke-Api "https://api.github.com/orgs/microsoft/members/$login" $env:GITHUB_TOKEN -ExpectedStatus 204 + } + 'Queue' { + if ($env:PIPELINE_ID -cnotmatch '\A[1-9][0-9]*\z') { throw 'Configure FSHARP_APEX_PIPELINE_ID in the fsharp_pr_validation environment.' } + if ([string]::IsNullOrWhiteSpace($env:AZDO_TOKEN)) { throw 'Azure did not return an access token.' } + $request = ConvertFrom-Json -InputObject $env:REQUEST_BODY -NoEnumerate + $run = Invoke-Api "https://dev.azure.com/devdiv/DevDiv/_apis/pipelines/$env:PIPELINE_ID/runs?api-version=7.1" ` + $env:AZDO_TOKEN -Method POST -Body $request -ExpectedStatus (200..299) + if ($run -is [array] -or $run.PSObject.Properties.Name -cnotcontains 'id' -or !(Test-SafeId $run.id)) { + throw 'Azure DevOps did not return a valid run ID.' + } + Add-Content -LiteralPath $env:GITHUB_OUTPUT -Encoding utf8 -Value "url=https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=$($run.id)" + } + 'Report' { + $workflowUrl = "$env:GITHUB_SERVER_URL/dotnet/fsharp/actions/runs/$env:GITHUB_RUN_ID" + $body = if ($env:RUN_URL) { + "[F# Apex run]($env:RUN_URL) requested by @$($event.comment.user.login).`n`nReviewed PR head and pipeline YAML: ``$env:HEAD_SHA```nBase: ``$env:BASE_SHA```n`nThe run verifies the PR merge against this snapshot before building. Changes require review and a new ``/dart `` or ``/pr-val `` comment." + } + else { + "F# Apex request failed. [Workflow details]($workflowUrl). Use a standalone ``/dart `` or ``/pr-val `` with the full current PR head SHA after reviewing code and pipeline YAML. The PR must target ``main`` and the requester needs repository write access and Microsoft-org membership." + } + $null = Invoke-Api "$repoApi/issues/$prNumber/comments" $env:GITHUB_TOKEN -Method POST -Body @{ body = $body } -ExpectedStatus 201 + } +} diff --git a/.github/workflows/pr-validation.yml b/.github/workflows/pr-validation.yml new file mode 100644 index 00000000000..a1ad5ad5f21 --- /dev/null +++ b/.github/workflows/pr-validation.yml @@ -0,0 +1,76 @@ +name: PR Validation (F# VS Integration) + +on: + issue_comment: + types: [created] + +permissions: + contents: read + pull-requests: write + id-token: write + +jobs: + validate-and-trigger: + if: | + github.repository == 'dotnet/fsharp' && + github.event.issue.pull_request && + (contains(github.event.comment.body, '/pr-val') || contains(github.event.comment.body, '/dart')) + runs-on: ubuntu-latest + environment: fsharp_pr_validation + defaults: + run: + shell: pwsh + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + # issue_comment's SHA is on the default branch, never the PR branch. + ref: ${{ github.sha }} + persist-credentials: false + + - name: Validate request and capture PR revision + id: request + env: + GITHUB_TOKEN: ${{ github.token }} + run: .\.github\scripts\pr-validation.ps1 -Mode Prepare + + - name: Get Microsoft membership lookup token + id: membership-token + uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2 + with: + app-id: ${{ secrets.MICROSOFT_MEMBERS_APP_ID }} + private-key: ${{ secrets.MICROSOFT_MEMBERS_APP_PRIVATE_KEY }} + owner: microsoft + permission-members: read + + - name: Verify Microsoft organization membership + env: + GITHUB_TOKEN: ${{ steps.membership-token.outputs.token }} + run: .\.github\scripts\pr-validation.ps1 -Mode Membership + + - name: Azure Login with OpenID Connect + uses: azure/login@8216e11d8cd9b42fe925c852af8e76311ff067ac # v2 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + allow-no-subscriptions: true + + - name: Queue DartLab pipeline + id: queue + env: + PIPELINE_ID: ${{ vars.FSHARP_APEX_PIPELINE_ID }} + REQUEST_BODY: ${{ steps.request.outputs.body }} + run: | + $ErrorActionPreference = 'Stop' + $env:AZDO_TOKEN = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv + if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($env:AZDO_TOKEN)) { throw 'Azure did not return an access token.' } + Write-Output "::add-mask::$env:AZDO_TOKEN" + .\.github\scripts\pr-validation.ps1 -Mode Queue + + - name: Report request result + if: success() || failure() + env: + GITHUB_TOKEN: ${{ github.token }} + RUN_URL: ${{ steps.queue.outputs.url }} + HEAD_SHA: ${{ steps.request.outputs.head }} + BASE_SHA: ${{ steps.request.outputs.base }} + run: .\.github\scripts\pr-validation.ps1 -Mode Report diff --git a/azure-pipelines-integration-dartlab.yml b/azure-pipelines-integration-dartlab.yml new file mode 100644 index 00000000000..76e7c66b39b --- /dev/null +++ b/azure-pipelines-integration-dartlab.yml @@ -0,0 +1,56 @@ +# Register in DevDiv with GitHub dotnet/fsharp as self. See eng/pipelines/apex-integration/README.md. +trigger: none +pr: none + +resources: + pipelines: + - pipeline: VisualStudioBuildUnderTest + source: DD-CB-ReleaseVS + trigger: none + repositories: + - repository: DartLab + type: git + name: DevDiv/DartLab + ref: refs/tags/Production + - repository: VSTemplates + type: git + name: DevDiv/VS.Templates + ref: refs/heads/main + +parameters: +- name: prNumber + type: string +- name: headSha + type: string +- name: baseSha + type: string +- name: visualStudioBootstrapperURI + type: string + default: '(default)' +- name: testMachineLab + displayName: DartLab-1ES Test Machine Lab + type: string + default: DotNet-Project-System-AMD64 +- name: testMachineHoldStrategy + displayName: Test Machine Hold Strategy + type: string + default: None + values: + - None + - Completion + - Failure + - Timeout + +extends: + template: /DartLab/1ES/pipelines/ci/build.yml@VSTemplates + parameters: + dartLabTemplatesRepositoryAlias: DartLab + testStages: + - template: /eng/pipelines/apex-integration/stage.yml@self + parameters: + prNumber: ${{ parameters.prNumber }} + headSha: ${{ parameters.headSha }} + baseSha: ${{ parameters.baseSha }} + visualStudioBootstrapperURI: ${{ parameters.visualStudioBootstrapperURI }} + testMachineLab: ${{ parameters.testMachineLab }} + testMachineHoldStrategy: ${{ parameters.testMachineHoldStrategy }} diff --git a/docs/release-notes/.VisualStudio/18.vNext.md b/docs/release-notes/.VisualStudio/18.vNext.md index 9eecbcbd147..2d032095814 100644 --- a/docs/release-notes/.VisualStudio/18.vNext.md +++ b/docs/release-notes/.VisualStudio/18.vNext.md @@ -1,5 +1,6 @@ ### Added +* Add maintainer-requested F# Apex Visual Studio integration tests on DartLab-1ES CloudTest, using PR pipeline definitions pinned to an explicitly requested commit SHA. ([PR #20625](https://github.com/dotnet/fsharp/pull/20625)) * Code-fixes for FS3888 (compiler-semantic attribute on the `.fs` but not the `.fsi`): copy the attribute into the `.fsi`, or remove it from the `.fs`. ([Issue #19560](https://github.com/dotnet/fsharp/issues/19560), [PR #19880](https://github.com/dotnet/fsharp/pull/19880)) * Expand `` in IDE tooltips, completion, and signature help, inheriting XML documentation from base classes, interfaces, overridden members, and constructors. ([Issue #19175](https://github.com/dotnet/fsharp/issues/19175), [PR #19188](https://github.com/dotnet/fsharp/pull/19188)) * Added a **Tools > Options > F# Tools > Compiler** option, **"Use the .NET SDK F# compiler for builds"**, on by default. Visual Studio uses the .NET SDK F# compiler when the project supplies SDK paths, matching `dotnet build`. Otherwise, it uses the bundled .NET Framework compiler. Turn the option off to use the bundled compiler for all projects. Changes are saved on Apply or OK; Cancel discards unsaved changes. Command-line builds are unaffected. ([Issue #20484](https://github.com/dotnet/fsharp/issues/20484), [PR #20485](https://github.com/dotnet/fsharp/pull/20485)) diff --git a/eng/Build.ps1 b/eng/Build.ps1 index 3b34d6e9a94..d7053f7da5a 100644 --- a/eng/Build.ps1 +++ b/eng/Build.ps1 @@ -62,6 +62,7 @@ param ( [switch]$testIntegration, [switch]$testScripting, [switch]$testVs, + [switch]$testApex, [switch]$testAll, [switch]$testAllButIntegration, [switch]$testAllButIntegrationAndAot, @@ -136,6 +137,7 @@ function Print-Usage() { Write-Host " -testIntegration Run F# integration tests" Write-Host " -testScripting Run Scripting tests" Write-Host " -testVs Run F# editor unit tests" + Write-Host " -testApex Run F# Apex VS integration tests (requires -deployExtensions)" Write-Host " -testpack Verify built packages" Write-Host " -testAOT Run AOT/Trimming tests" Write-Host " -testEditor Run VS Editor tests" @@ -409,6 +411,245 @@ function TestUsingMSBuild([string] $testProject, [string] $targetFramework, [str Exec-Console $dotnetExe $test_args } +# Runs the Apex VS integration tests. These are a classic MSTest (VSTest) project, NOT part of the +# repo's Microsoft.Testing.Platform 'dotnet test' path, and they drive a real VS instance via the Apex +# UI-automation framework — so they are launched with vstest.console.exe (from the installed VS). +# The project is intentionally excluded from VisualFSharp.slnx (its dependencies must version-match the +# VS it drives), so this builds it explicitly. It also must match a specific VS version +# (MicrosoftTestApexVisualStudioVersion); if the installed VS does not match, the run is skipped rather +# than producing confusing Apex/VS-mismatch failures. The F# VSIX must be deployed into the RoslynDev +# hive first (build with -deployExtensions); Apex launches devenv /rootsuffix RoslynDev. + +# ===== BEGIN Apex CI diagnostics helpers (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== +# On CI the Apex host failed with "Timed out ... trying get running object": devenv started but never +# registered its DTE automation object in the ROT. These helpers (a) make sure devenv has an interactive +# console desktop (the usual reason DTE never registers) and (b) collect the RoslynDev-hive logs so we can +# tell an interactive-desktop problem apart from an F# VSIX / package load failure. +function Capture-ApexScreenshot([string] $path) { + # Returns $true only if a screenshot could be taken, which implies an interactive desktop is attached. + try { + Add-Type -AssemblyName System.Windows.Forms -ErrorAction Stop + Add-Type -AssemblyName System.Drawing -ErrorAction Stop + $bounds = [System.Windows.Forms.SystemInformation]::VirtualScreen + $bmp = New-Object System.Drawing.Bitmap $bounds.Width, $bounds.Height + $gfx = [System.Drawing.Graphics]::FromImage($bmp) + try { + $gfx.CopyFromScreen($bounds.Location, [System.Drawing.Point]::Empty, $bounds.Size) + $bmp.Save($path, [System.Drawing.Imaging.ImageFormat]::Png) + Write-Host "Captured screenshot to '$path'." + return $true + } finally { + $gfx.Dispose(); $bmp.Dispose() + } + } catch { + Write-Host "Screenshot capture failed (no interactive desktop?): $($_.Exception.Message)" + return $false + } +} + +function Prepare-ApexInteractiveSession([string] $screenshotPath) { + # Mirrors dotnet/roslyn's Setup-IntegrationTestRun: if the desktop is not interactive, reconnect the + # session to the console so devenv can publish its DTE automation object; then minimize other windows. + if (-not (Capture-ApexScreenshot $screenshotPath)) { + Write-Host "No interactive desktop detected; attempting to reconnect the session to the console." + try { + $quserItems = ((quser $env:USERNAME | Select-Object -Skip 1) -split '\s+') + $sessionId = $quserItems[2] + if ($sessionId -eq 'Disc') { $sessionId = $quserItems[1] } + Write-Host "tscon $sessionId /dest:console" + tscon $sessionId /dest:console + Start-Sleep -Seconds 3 + [void](Capture-ApexScreenshot $screenshotPath) + } catch { + Write-Host "##vso[task.logissue type=warning]Failed to reconnect session to console: $($_.Exception.Message)" + } + } + + try { + (New-Object -ComObject "Shell.Application").MinimizeAll() + } catch { + Write-Host "MinimizeAll failed: $($_.Exception.Message)" + } +} + +function Collect-ApexDiagnostics([string] $screenshotPath, [string] $destination) { + # After-run screenshot plus the RoslynDev experimental-hive logs. ActivityLog.xml records whether the + # F# package failed to load; ComponentModelCache\*.err records MEF composition failures. + [void](Capture-ApexScreenshot $screenshotPath) + + $roamingVs = Join-Path $env:USERPROFILE "AppData\Roaming\Microsoft\VisualStudio" + $localVs = Join-Path $env:USERPROFILE "AppData\Local\Microsoft\VisualStudio" + + if (Test-Path $roamingVs) { + foreach ($hiveDir in @(Get-ChildItem -Path $roamingVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) { + foreach ($name in @("ActivityLog.xml", "ActivityLog.xsl")) { + $src = Join-Path $hiveDir.FullName $name + if (Test-Path $src) { + Copy-Item $src (Join-Path $destination "$($hiveDir.Name)-$name") -Force -ErrorAction SilentlyContinue + Write-Host "Collected $src" + } + } + } + } + if (Test-Path $localVs) { + foreach ($hiveDir in @(Get-ChildItem -Path $localVs -Directory -Filter "*RoslynDev" -ErrorAction SilentlyContinue)) { + $mefErr = Join-Path $hiveDir.FullName "ComponentModelCache\Microsoft.VisualStudio.Default.err" + if (Test-Path $mefErr) { + Copy-Item $mefErr (Join-Path $destination "$($hiveDir.Name)-Microsoft.VisualStudio.Default.err") -Force -ErrorAction SilentlyContinue + Write-Host "Collected $mefErr" + } + } + } +} +# ===== END Apex CI diagnostics helpers (Phase 2) ===== + +function TestApexUsingVSTest([string] $targetFramework) { + $projectName = "FSharp.Editor.Apex.IntegrationTests" + $apexProject = Join-Path $RepoRoot "vsintegration\tests\$projectName\$projectName.csproj" + + # Create the results directory up-front so the CI 'publish test logs' step never fails on a missing + # path, even when the run is skipped below (a marker file records the skip reason). + $testResultsDir = "$ArtifactsDir\TestResults\$configuration" + Create-Directory $testResultsDir + + # The Apex package (MicrosoftTestApexVisualStudioVersion) is pinned to a VS 18.x build. Because the + # installed VS differs by machine (CI's image trails a dev's local install and they rarely share the + # same minor), gate on the MAJOR version by default so any VS 18.x can run the tests; set + # FSHARP_APEX_VS_VERSION to a stricter prefix (e.g. "18.9") to force an exact minor. + $expectedVsVersion = if (${env:FSHARP_APEX_VS_VERSION}) { ${env:FSHARP_APEX_VS_VERSION} } else { "18" } + + # Determine which VS will run the tests, and gate on its version. An explicit + # VisualStudio.InstallationUnderTest.Path override is trusted (gate skipped); otherwise enumerate + # ALL installed VS instances (not just the latest) and pick one whose version starts with the target + # prefix, skipping the run if none match rather than producing confusing Apex/VS-mismatch errors. + $overridePath = ${env:VisualStudio.InstallationUnderTest.Path} + if ($overridePath) { + Write-Host "Using explicit VisualStudio.InstallationUnderTest.Path=$overridePath (VS version gate skipped)." + $vsDir = Split-Path -Parent (Split-Path -Parent (Split-Path -Parent $overridePath)) + } + else { + $vswhere = Join-Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" + # Query the properties separately (parallel arrays in a stable order) instead of parsing + # `-format json`: Windows PowerShell 5.1's ConvertFrom-Json mishandles vswhere's multi-instance + # JSON array (it collapses the instances into one object with array-valued properties), which + # would make every version comparison fail. Build.ps1 runs under Windows PowerShell 5.1. + $versions = @() + $paths = @() + $names = @() + if (Test-Path $vswhere) { + $versions = @(& $vswhere -all -prerelease -products * -property installationVersion) + $paths = @(& $vswhere -all -prerelease -products * -property installationPath) + $names = @(& $vswhere -all -prerelease -products * -property displayName) + } + $found = $versions -join ', ' + + # Print the full VS inventory for diagnostics (helps explain a skip on CI). Always logged. + Write-Host "Visual Studio instances found by vswhere ($($versions.Count)) [vswhere: $vswhere]:" + if ($versions.Count -eq 0) { + Write-Host " (none)" + } + for ($k = 0; $k -lt $versions.Count; $k++) { + $mm = ($versions[$k] -split '\.')[0..1] -join '.' + $hasIde = Test-Path (Join-Path $paths[$k] "Common7\IDE\devenv.exe") + $name = if ($k -lt $names.Count) { $names[$k] } else { "" } + Write-Host (" [{0}] version={1} (major.minor {2}) devenv={3} name='{4}' path={5}" -f $k, $versions[$k], $mm, $hasIde, $name, $paths[$k]) + } + + # Among installs whose version starts with the target prefix (major, or an explicit major.minor) + # and that actually have an IDE (devenv.exe — excludes Build Tools / incomplete installs), pick + # the LOWEST version: it is closest to the pinned Apex minor, and "Apex older than VS" is the + # safer cross-minor direction than the reverse. + $vsDir = $null + $selectedVersion = $null + $versionMatchPath = $null + $candidates = for ($k = 0; $k -lt $versions.Count; $k++) { + if ($versions[$k] -like "$expectedVsVersion.*") { + [PSCustomObject]@{ Version = $versions[$k]; Path = $paths[$k] } + } + } + $candidates = @($candidates | Sort-Object { [version]$_.Version }) + foreach ($c in $candidates) { + if (-not $versionMatchPath) { $versionMatchPath = $c.Path } + if (Test-Path (Join-Path $c.Path "Common7\IDE\devenv.exe")) { + $vsDir = $c.Path.TrimEnd("\") + $selectedVersion = $c.Version + break + } + } + + if (-not $vsDir) { + if ($versionMatchPath) { + Write-Host "##vso[task.logissue type=warning]Found VS $expectedVsVersion ($versionMatchPath) but no devenv.exe under Common7\IDE (incomplete install, Build Tools, or a VS update in progress); skipping Apex integration tests (non-gating)." + Write-Host "Skipping Apex tests: VS $expectedVsVersion present but devenv.exe missing at [$versionMatchPath]. If VS was updating, retry once it finishes." + } + else { + Write-Host "##vso[task.logissue type=warning]No installed VS matches the Apex target $expectedVsVersion (found: $found); skipping Apex integration tests (non-gating)." + Write-Host "Skipping Apex tests: no VS $expectedVsVersion found among installs [$found]. Set FSHARP_APEX_VS_VERSION or VisualStudio.InstallationUnderTest.Path to run against a different install." + } + Set-Content -Path (Join-Path $testResultsDir "apex-tests-skipped.txt") -Value "Apex integration tests skipped: no usable VS matching target $expectedVsVersion (installed: $found)." + return + } + Write-Host "Selected VS $selectedVersion at '$vsDir' for the Apex tests (target $expectedVsVersion)." + # This process env var propagates to the child vstest.console -> testhost -> Apex processes. + ${env:VisualStudio.InstallationUnderTest.Path} = Join-Path $vsDir "Common7\IDE\devenv.exe" + } + + $vstestConsole = @( + (Join-Path $vsDir "Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"), + (Join-Path $vsDir "Common7\IDE\Extensions\TestPlatform\vstest.console.exe") + ) | Where-Object { Test-Path $_ } | Select-Object -First 1 + if (-not $vstestConsole) { + throw "vstest.console.exe not found under '$vsDir' (looked in CommonExtensions\Microsoft\TestWindow and Extensions\TestPlatform)." + } + + # Build the Apex test project explicitly (it is intentionally excluded from VisualFSharp.slnx). + $dotnetPath = InitializeDotNetCli + $dotnetExe = Join-Path $dotnetPath "dotnet.exe" + $buildBinLog = "$LogDir\${projectName}_$targetFramework.binlog" + Exec-Console $dotnetExe "build `"$apexProject`" -c $configuration /bl:`"$buildBinLog`"" + + $testAssembly = Join-Path $ArtifactsDir "bin\$projectName\$configuration\$targetFramework\$projectName.dll" + if (-not (Test-Path $testAssembly)) { + throw "Apex test assembly not found at '$testAssembly' after build." + } + + $jobName = if ($env:SYSTEM_JOBNAME) { $env:SYSTEM_JOBNAME } else { "local" } + $trxFileName = "$projectName.$targetFramework.$jobName.trx" + + $vstestArgs = @( + $testAssembly, + "/Platform:x64", + "/Framework:.NETFramework,Version=v4.7.2", + "/logger:trx;LogFileName=$trxFileName", + "/ResultsDirectory:$testResultsDir" + ) + + Write-Host "$vstestConsole $($vstestArgs -join ' ')" + + # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== + if ($ci) { + Prepare-ApexInteractiveSession (Join-Path $testResultsDir "apex-before-run.png") + } + # ===== END Apex CI diagnostics (Phase 2) ===== + + & $vstestConsole @vstestArgs + $vstestExitCode = $LASTEXITCODE + + # ===== BEGIN Apex CI diagnostics (Phase 2, TEMPORARY — revert once the CI hang is understood) ===== + if ($ci) { + Collect-ApexDiagnostics (Join-Path $testResultsDir "apex-after-run.png") $testResultsDir + } + # ===== END Apex CI diagnostics (Phase 2) ===== + + # This leg is non-gating (signal only): a test failure must NOT fail the build. Surface it as a + # warning and let the published TRX / Tests tab carry the signal. Genuine infra errors (missing + # assembly / vstest.console) already threw above and remain hard failures. + if ($vstestExitCode -ne 0) { + Write-Host "##vso[task.logissue type=warning]Apex integration tests reported failures (vstest.console exit code $vstestExitCode). This leg is non-gating; see the published TRX and the Tests tab." + Write-Host "Apex tests exit code $vstestExitCode (non-gating; not failing the build)." + } +} + function TestFromManifest([string] $platform, [string] $selection = "--all") { $dotnetPath = InitializeDotNetCli $dotnetExe = Join-Path $dotnetPath "dotnet.exe" @@ -682,6 +923,10 @@ try { TestUsingMSBuild -testProject "$RepoRoot\vsintegration\tests\FSharp.Editor.IntegrationTests\FSharp.Editor.IntegrationTests.csproj" -targetFramework $script:desktopTargetFramework } + if ($testApex) { + TestApexUsingVSTest -targetFramework $script:desktopTargetFramework + } + if ($testAOT) { Push-Location "$RepoRoot\tests\AheadOfTime" ./check.ps1 diff --git a/eng/Packages.props b/eng/Packages.props index eac871c2d58..fb6753942db 100644 --- a/eng/Packages.props +++ b/eng/Packages.props @@ -52,6 +52,7 @@ + @@ -100,6 +101,8 @@ + + diff --git a/eng/SetupVSHive.ps1 b/eng/SetupVSHive.ps1 index 50dcf7531d0..b6ec083d97f 100644 --- a/eng/SetupVSHive.ps1 +++ b/eng/SetupVSHive.ps1 @@ -12,6 +12,46 @@ $vsRegEdit = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'VSRegEdit $hive = "RoslynDev" &$vsRegEdit set "$vsDir" $hive HKCU "Roslyn\Internal\OnOff\Features" OOP64Bit dword 0 +# Suppress modal dialogs and online-profile prompts that block UI automation (Apex) and cause the +# integration tests to hang until they time out. These mirror the settings dotnet/roslyn applies to its +# RoslynDev hive before running VS integration tests. + +# Disable the editor "report exceptions" dialog: fail silently and keep the test running instead of +# popping a modal that stalls automation. +&$vsRegEdit set "$vsDir" $hive HKCU "Text Editor" "Report Exceptions" dword 0 + +# Disable roaming settings so an online user profile / sign-in prompt can't interfere on CI. +&$vsRegEdit set "$vsDir" $hive HKCU "ApplicationPrivateSettings\Microsoft\VisualStudio" RoamingEnabled string "1*System.Boolean*False" + +# Disable background download UI to avoid toasts appearing over the IDE during a run. +&$vsRegEdit set "$vsDir" $hive HKCU "FeatureFlags\Setup\BackgroundDownload" Value dword 0 + +# Disable targeted (remote settings) notifications. This one can't be set via VsRegEdit, so write it +# directly to the user hive (it is not hive-suffix specific). +reg add hkcu\Software\Microsoft\VisualStudio\RemoteSettings /f /t REG_DWORD /v TurnOffSwitch /d 1 | Out-Null + +# Initialize the experimental hive's first-launch state non-interactively. On a fresh hive devenv shows a +# full-screen "Sign in to Visual Studio" first-launch experience that blocks the UI thread, so it never +# registers its DTE automation object and Apex times out waiting for it. Importing settings via +# /resetsettings completes first launch without that dialog. This mirrors what the +# Microsoft.VisualStudio.Extensibility.Testing harness does before running VS integration tests on CI. + +# Start VS with an empty environment (no start window / start page) on launch. +&$vsRegEdit set "$vsDir" $hive HKCU "General" OnEnvironmentStartup dword 10 + +$devenv = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.exe' +$firstLaunchLog = Join-Path $env:TEMP "vs-firstlaunch-$hive.log" +Write-Host "Initializing '$hive' hive first launch: devenv /rootsuffix $hive /resetsettings General.vssettings /command File.Exit" +$devenvArgs = @("/rootsuffix", $hive, "/resetsettings", "General.vssettings", "/command", "File.Exit", "/logFile:$firstLaunchLog") +$proc = Start-Process -FilePath $devenv -ArgumentList $devenvArgs -PassThru +if (-not $proc.WaitForExit(180000)) { + Write-Host "##vso[task.logissue type=warning]devenv first-launch init did not exit within 180s; terminating it." + try { Stop-Process -Id $proc.Id -Force } catch {} +} +else { + Write-Host "devenv first-launch init exited with code $($proc.ExitCode)." +} + Write-Host "-- VS Info --" $isolationIni = Join-Path (Join-Path (Join-Path $vsDir 'Common7') 'IDE') 'devenv.isolation.ini' Get-Content $isolationIni | Write-Host diff --git a/eng/Versions.props b/eng/Versions.props index 7bcf4b16972..11277575b45 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -103,6 +103,11 @@ 18.9.179 0.1.800-beta + + 18.7.0-preview-1-11715-043 + 3.6.4 + 17.14.2120 diff --git a/eng/pipelines/apex-integration/README.md b/eng/pipelines/apex-integration/README.md new file mode 100644 index 00000000000..7c5a141dd1d --- /dev/null +++ b/eng/pipelines/apex-integration/README.md @@ -0,0 +1,133 @@ +# F# Apex integration tests on DartLab-1ES + +This pipeline installs Visual Studio from `DD-CB-ReleaseVS` on a DartLab-1ES +CloudTest machine, builds and deploys the PR's F# VSIX, then runs the Apex tests in Release. +It avoids the dependency skew of older preinstalled CI images. The chosen VS drop +must still support the PR's package pins; a recent drop is not a compatibility guarantee. + +## Requesting a run + +Post **`/dart `** or **`/pr-val `** as the entire comment on an open +`dotnet/fsharp` PR targeting `main`. Replace `` with the full 40-character +lowercase SHA of the current PR head after reviewing that revision, including its +pipeline YAML, templates and scripts. Short, stale or non-head SHAs are rejected. +Both commands run the same pipeline. Fork PRs are supported. +The commenter must have repository write, maintain or admin access **and** be a +member of the `microsoft` GitHub organization. Supplying the SHA is the requester's +explicit authorization of that revision, not an automatic check of GitHub review status. + +The [GitHub workflow](../../../.github/workflows/pr-validation.yml) captures the PR +head and base SHAs and queues Azure DevOps over OIDC. It posts a run link and those +revisions to the PR. New pushes, PR events and VS builds do **not** request runs. +Administrators can still queue runs manually, subject to pipeline permissions. + +The [pipeline](../../../azure-pipelines-integration-dartlab.yml) is registered +directly against GitHub `dotnet/fsharp`; its `self` is **not** the dnceng mirror. +The Actions workflow and authorization helper run from the default branch. +The workflow pins pipeline YAML and local templates to `refs/pull//head` +at the explicitly requested head SHA, including for fork PRs. On the test machine, +[`setup-pr-validation.ps1`](../../setup-pr-validation.ps1) fetches the public PR +merge ref from a checkout of that reviewed head, verifies both parents against +the captured head/base SHAs, checks the PR is still open and observes current +head/base refs before checking out that exact merge. + +A conflict, missing/stale merge, changed head/base, or API/Git failure stops the run +before building PR code in the provided pipeline. Review the updated head and +post a new command with its SHA after resolving the problem. A push +after verification does not alter the detached commit being tested. The setup and +test logs record the actual merge SHA; Azure DevOps `Build.SourceVersion` identifies +the reviewed PR head/YAML revision, **not** the tested PR merge. Results apply only +to the recorded snapshot, not automatically to later PR revisions. + +## Registration and authorization + +Provision these before enabling requests: + +1. In **devdiv/DevDiv -> Pipelines -> New pipeline -> GitHub**, select `dotnet/fsharp` + and the existing `azure-pipelines-integration-dartlab.yml`, default branch `main`. + Use an approved Azure Pipelines GitHub App connection restricted to the necessary + repository; authorize this pipeline specifically. No `dnceng-internal-code-access` + connection is needed. Leave the existing mirror and dnceng pipelines unchanged. +2. Authorize `DevDiv/DartLab`, `DevDiv/VS.Templates`, the `DD-CB-ReleaseVS` + pipeline/artifacts, required feeds, and the `DartLab-1ES` service connection. + The stage uses the production CloudTest templates from `DevDiv/DartLab`; it has + no dependency on the legacy `DevDiv/DartLab.Templates` DTL repository. +3. The DartLab team assigned the F# pipeline to the + **`DotNet-Project-System-AMD64`** CloudTest lab. The owner notification alias + is `fsharp`. Confirm the VS drop and installed components support F#. +4. Create the GitHub **`fsharp_pr_validation`** environment. Set its variable + **`FSHARP_APEX_PIPELINE_ID`** to the new DevDiv pipeline's numeric ID. + Set **`AZURE_CLIENT_ID`** and **`AZURE_TENANT_ID`** secrets for an Entra identity + federated to `repo:dotnet/fsharp:environment:fsharp_pr_validation`. + Grant it permission to queue this pipeline, not edit pipeline definitions. + This flow does not require an Azure subscription. +5. Provide **`MICROSOFT_MEMBERS_APP_ID`** and **`MICROSOFT_MEMBERS_APP_PRIVATE_KEY`** + environment secrets for an approved GitHub App installed in `microsoft`, with + organization **Members: read** permission. The workflow requests a short-lived + membership-only token. The dotnet/fsharp `GITHUB_TOKEN` alone cannot be assumed + to see private memberships in another organization. Verify lookup with a private + member before rollout. Lookup errors fail closed; they do not prove nonmembership. +6. Have the lab owners approve execution of fork PR code and verify effective + credential/network isolation. This is a manually queued build that executes + reviewed PR pipeline YAML: **do not assume automatic fork-build secret + restrictions apply**. PR YAML can change steps and resource usage before the + provided source-verification script runs, so the SHA gate is not a sandbox. + Review job tokens, feed credentials, service connections and internal + network access. Git fetches use the public repository, and the Actions OIDC and + membership tokens are not sent to the test machine, but these measures alone do + not sandbox a build. Do not enable this pipeline if lab policy cannot support it. + +These are three separate identities: the Azure Pipelines GitHub source connection, +the Entra queue identity, and the Microsoft-org membership lookup App. None replaces +the others. Protect `main`, restrict who can edit/queue this privileged pipeline, +keep `testMachineHoldStrategy: None` as the default, and authorize no +signing/publishing credentials. Use `Failure` or `Completion` only for a bounded +investigation; CloudTest held machines are accessed through the generated Bastion link. + +Microsoft supports [GitHub pipelines in multiple Azure DevOps organizations][github]. +Only the first organization receives automatic GitHub push/PR triggers; manual queueing +remains supported in secondary organizations. This workflow uses the [Runs REST API][runs], +so the restriction does not require moving existing dnceng CI or enabling PR triggers. + +## Validation and activation + +Local regression commands (no product build or DartLab access required): + +```powershell +pwsh -NoProfile -File .github\scripts\pr-validation.Tests.ps1 +powershell -NoProfile -ExecutionPolicy Bypass -File eng\tests\SetupPrValidation.Tests.ps1 +``` + +The script tests use temporary local Git repositories and a mocked GitHub PR response. +They cover merge identity, ref races, closed/changed PRs and failure before checkout. +The PowerShell 7 tests run the Actions helper offline with a fake HTTP boundary. +They cover command syntax, authorization of the original commenter (also on reruns), +private membership lookup with the separate App token, the queued snapshot and PR feedback. + +After provisioning, use the Runs API's `previewRun` with the same repository +ref/version, snapshot parameters, `DD-CB-ReleaseVS` resource pin and confirmed +`testMachineLab` to verify actual private-template expansion. Fix every template, +parameter, expression, service-connection and operation-expansion error before +allocating a CloudTest machine. Verify GitHub `self.version` pinning against the +requested head SHA for same-repository and fork PRs, and the test-machine source +bootstrap; local YAML parsing cannot establish either. Land the Actions +workflow/authorization helper on main before activation, and configure the +environment and real pipeline ID together. + +Explicitly request same-repository and fork PR smoke runs. Confirm the exact merge +SHA, installed VS, VSIX load, Azure DevOps test counts, Apex results and CloudTest +TRX/build/setup log artifacts. Exercise changed head/base rejection and an +unauthorized request that queues no run. Exercise missing, short, mismatched and +stale command SHAs. Verify the compiled YAML/local templates come from the +explicitly requested PR head rather than main or a newer push, and no UI overrides, +schedules or resource triggers introduce automatic runs. Completion is not reported +back as a new required GitHub check; the PR comment links to DevDiv results. + +Source-connection precedent: [NuGet/NuGet.Client#5936][nuget] introduced a self-contained +VS-test pipeline for manual PR-branch queueing. F# retains its VS-drop selection and +Apex build path while using DartLab-1ES CloudTest operations rather than legacy DTL +agent step lists. + +[github]: https://learn.microsoft.com/en-us/azure/devops/pipelines/repos/github?view=azure-devops +[runs]: https://learn.microsoft.com/en-us/rest/api/azure/devops/pipelines/runs/run-pipeline?view=azure-devops-rest-7.1 +[nuget]: https://github.com/NuGet/NuGet.Client/pull/5936 diff --git a/eng/pipelines/apex-integration/run-apex.ps1 b/eng/pipelines/apex-integration/run-apex.ps1 new file mode 100644 index 00000000000..65515b73669 --- /dev/null +++ b/eng/pipelines/apex-integration/run-apex.ps1 @@ -0,0 +1,24 @@ +[CmdletBinding()] +param( + [ValidateSet('Debug', 'Release')] + [string] $Configuration = 'Release' +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$repositoryRoot = Resolve-Path (Join-Path $PSScriptRoot '..\..\..') +Set-Location $repositoryRoot + +$mergeSha = (& git rev-parse HEAD).Trim() +if ($LASTEXITCODE -ne 0 -or $mergeSha -notmatch '^[0-9a-f]{40}$') { + throw 'Unable to determine the verified PR merge SHA.' +} +Write-Host "Running Apex against verified PR merge $mergeSha." + +$env:NativeToolsOnMachine = 'true' +& .\eng\SetupVSHive.ps1 +& .\eng\CIBuildNoPublish.cmd -configuration $Configuration -deployExtensions -testApex +if ($LASTEXITCODE -ne 0) { + throw "Apex build and tests failed with exit code $LASTEXITCODE." +} diff --git a/eng/pipelines/apex-integration/stage.yml b/eng/pipelines/apex-integration/stage.yml new file mode 100644 index 00000000000..6587c7089d3 --- /dev/null +++ b/eng/pipelines/apex-integration/stage.yml @@ -0,0 +1,161 @@ +parameters: +- name: prNumber + type: string +- name: headSha + type: string +- name: baseSha + type: string +- name: visualStudioBootstrapperURI + type: string + default: '(default)' +- name: testMachineLab + type: string + default: DotNet-Project-System-AMD64 +- name: testMachineHoldStrategy + type: string + default: None + +stages: +- template: /templates/stages/single-executable.yml@DartLab + parameters: + name: FSharpVSIntegration + displayName: F# VS Integration (Apex) + owner: fsharp + testMachineLab: ${{ parameters.testMachineLab }} + testMachineHoldStrategy: ${{ parameters.testMachineHoldStrategy }} + templateContext: + owner: fsharp + areaPath: DevDiv\NET Developer Experience\FSharp + testExecutablePath: C:\Windows\System32\WindowsPowerShell\v1.0\powershell.exe + testExecutableArguments: >- + -NoProfile + -ExecutionPolicy Bypass + -File C:\Test\FSharp\eng\pipelines\apex-integration\run-apex.ps1 + -Configuration Release + testRunName: F# Apex integration tests + testRunTimeout: 0:4:0:0 + testResultType: TRX + testMachineEnvironmentVariables: + NativeToolsOnMachine: 'true' + VisualStudio.InstallationUnderTest.Path: C:\Test\VisualStudio\Common7\IDE\devenv.exe + testMachineTotalCount: '1' + testMachineFilesToUpload: + - CopyDirectory: C:\Test\FSharp\artifacts + Subdirectory: FSharp + IncludeFilePatterns: + - TestResults\Release\**\* + - log\Release\**\* + ExcludeFilePatterns: [] + Timeout: 0:0:10:0 + preTestMachineConfigurationSteps: + - checkout: none + - task: DownloadBuildArtifacts@1 + displayName: Download Visual Studio build artifacts + inputs: + buildType: specific + project: $(resources.pipeline.VisualStudioBuildUnderTest.projectID) + definition: $(resources.pipeline.VisualStudioBuildUnderTest.pipelineID) + buildVersionToDownload: specific + buildId: $(resources.pipeline.VisualStudioBuildUnderTest.runID) + downloadType: specific + artifactName: BuildArtifacts + downloadPath: $(Pipeline.Workspace)\VisualStudioBuildUnderTest + - task: PowerShell@2 + name: SetProductsDropName + displayName: Set 'VisualStudio.BuildUnderTest.ProductsDropName' + inputs: + filePath: $(DartLab.Path)\Scripts\VisualStudio\Build\Get-VisualStudioDropName.ps1 + arguments: -DropNamePrefix 'Products' -VstsDropUrlsJson '$(Pipeline.Workspace)\VisualStudioBuildUnderTest\BuildArtifacts\VstsDropUrls.json' -OutVariableName 'VisualStudio.BuildUnderTest.ProductsDropName' + preOperationsSteps: + - template: /templates/steps/powershell/execute-script.yml@DartLab + parameters: + displayName: Set 'VisualStudio.InstallationUnderTest.Signing' + filePath: $(DartLab.Path)\Scripts\VisualStudio\Signing\Get-VisualStudioSigning.ps1 + arguments: -BootstrapperURI '${{ iif(eq(parameters.visualStudioBootstrapperURI, '(default)'), 'https://vsdrop.corp.microsoft.com/file/v1/$(VisualStudio.BuildUnderTest.ProductsDropName);bootstrappers/Enterprise/vs_enterprise.exe', parameters.visualStudioBootstrapperURI) }}' -OutVariableName 'VisualStudio.InstallationUnderTest.Signing' -ServiceConnectionTenantID $env:AZURESUBSCRIPTION_TENANT_ID -ServiceConnectionClientID $env:AZURESUBSCRIPTION_CLIENT_ID -ServiceConnectionID $env:AZURESUBSCRIPTION_SERVICE_CONNECTION_ID + serviceConnectionName: DartLab-1ES + checkpointOperations: + - operation: executePowerShellScript + displayName: Initialize Visual Studio Automation + scriptPath: C:\DartLab\Scripts\VisualStudio\Initialize-VisualStudioAutomation.ps1 + scriptParameters: -Signing '$(VisualStudio.InstallationUnderTest.Signing)' -LogsDirectory '[LoggingDirectory]' + timeout: 0:0:7:0 + - operation: checkpoint + displayName: Initialize Visual Studio Environment Checkpoint + capabilities: + CheckpointID: d1c386cf-b987-4186-829f-405d45e97ef3 + Signing: $(VisualStudio.InstallationUnderTest.Signing) + preDeployAndRunTestsOperations: + - operation: executePowerShellScript + displayName: Install VSTest + scriptPath: C:\DartLab\Scripts\Test\Platform\Install-VisualStudioTest.ps1 + scriptParameters: -DestinationDirectory 'C:\VSTest' -Version 'latest' -OutputEnvironmentVariableName 'DartVSTestExecutablePath' + timeout: 0:0:10:0 + - operation: executePowerShellScript + displayName: Install Visual Studio Under Test + scriptPath: C:\DartLab\Scripts\VisualStudio\Installation\Install-VisualStudio.ps1 + scriptParameters: >- + -BootstrapperURI '${{ iif(eq(parameters.visualStudioBootstrapperURI, '(default)'), 'https://vsdrop.corp.microsoft.com/file/v1/$(VisualStudio.BuildUnderTest.ProductsDropName);bootstrappers/Enterprise/vs_enterprise.exe', parameters.visualStudioBootstrapperURI) }}' + -InstallationParameters '--add Microsoft.VisualStudio.Component.CoreEditor --add Microsoft.VisualStudio.Workload.CoreEditor --add Microsoft.Component.MSBuild --add Microsoft.NetCore.Component.SDK --add Microsoft.NetCore.Component.Runtime.8.0 --add Microsoft.NetCore.Component.DevelopmentTools --add Microsoft.Net.Component.4.7.2.TargetingPack --add Microsoft.Net.Component.4.8.SDK --add Microsoft.VisualStudio.Component.Roslyn.Compiler --add Microsoft.VisualStudio.Component.Roslyn.LanguageServices --add Microsoft.VisualStudio.Component.FSharp --add Microsoft.VisualStudio.Component.FSharp.Desktop --add Microsoft.VisualStudio.Component.ManagedDesktop.Core --add Microsoft.VisualStudio.Component.ManagedDesktop.Prerequisites --add Microsoft.VisualStudio.Workload.ManagedDesktop --add Microsoft.Component.CodeAnalysis.SDK --add Microsoft.VisualStudio.Component.VSSDK --add Microsoft.VisualStudio.ComponentGroup.VisualStudioExtension.Prerequisites --add Microsoft.VisualStudio.Workload.VisualStudioExtension --installPath "C:\Test\VisualStudio" --quiet --norestart --wait' + -LogsDirectory '[LoggingDirectory]' + -ManagedIdentityClientID '$(DartLab.1ES.ManagedIdentity.ClientID)' + -Timeout '2:0:0' + timeout: 2:0:0 + timeoutBuffer: 0:0:5:0 + - operation: executePowerShellScript + displayName: Initialize Visual Studio User Automation + scriptPath: C:\DartLab\Scripts\VisualStudio\Initialize-VisualStudioUserAutomation.ps1 + scriptParameters: -UserName 'cloudtest' + timeout: 0:0:5:0 + - operation: executePowerShellScript + displayName: Initialize Visual Studio Telemetry + scriptPath: C:\DartLab\Scripts\VisualStudio\Initialize-VisualStudioTelemetry.ps1 + scriptParameters: -UserName 'cloudtest' + timeout: 0:0:7:0 + - operation: restartMachine + displayName: Restart after Visual Studio installation + - operation: executePowerShellScript + displayName: Validate Visual Studio Setup + scriptPath: C:\DartLab\Scripts\Test\Invoke-VisualStudioTest.ps1 + scriptParameters: -TestFileNames 'C:\DartLab\Tests\VisualStudio\DartLab.Tests.VisualStudio.dll' -TestAdapterPaths 'C:\DartLab\Tests' -Settings 'C:\DartLab\Tests\VisualStudio\VisualStudio.runsettings' -TestCaseFilter 'TestCategory=Default' -Diag '[LoggingDirectory]\Logs\Validate-VisualStudio\vsdiag.txt' -ResultsDirectory '[LoggingDirectory]\Logs\Validate-VisualStudio' -VSTestConsoleExePath $env:DartVSTestExecutablePath -TestRunParameters @{ ExpectedNumberOfVisualStudioInstallations = '1'; ResetPauseDuration = '00:00:00:20' } + environmentVariables: + VisualStudio.InstallationUnderTest.Path: C:\Test\VisualStudio + timeout: 0:0:10:0 + - operation: executePowerShellInline + displayName: Enable Long Paths + scriptBlock: reg add HKLM\SYSTEM\CurrentControlSet\Control\FileSystem /v LongPathsEnabled /t REG_DWORD /d 1 /f + timeout: 0:0:1:0 + - operation: executePowerShellInline + displayName: Verify and check out PR snapshot + scriptBlock: | + $ErrorActionPreference = 'Stop' + $sourceRoot = 'C:\Test\FSharp' + if (Test-Path -LiteralPath $sourceRoot) { + throw "$sourceRoot already exists; refusing to reuse an unverified checkout." + } + git init $sourceRoot + if ($LASTEXITCODE -ne 0) { throw 'git init failed.' } + git -C $sourceRoot remote add origin https://github.com/dotnet/fsharp.git + if ($LASTEXITCODE -ne 0) { throw 'git remote add failed.' } + if ($env:PIPELINE_SOURCE_SHA -cne $env:HEAD_SHA) { + throw 'The pipeline YAML revision does not match the reviewed PR head.' + } + git -C $sourceRoot fetch --no-tags --depth=1 origin $env:PIPELINE_SOURCE_SHA + if ($LASTEXITCODE -ne 0) { throw 'Fetching the reviewed pipeline source revision failed.' } + git -C $sourceRoot checkout --detach FETCH_HEAD + if ($LASTEXITCODE -ne 0) { throw 'Checking out the reviewed pipeline source revision failed.' } + if ((git -C $sourceRoot rev-parse HEAD).Trim() -cne $env:PIPELINE_SOURCE_SHA) { + throw 'The reviewed source checkout does not match Build.SourceVersion.' + } + Push-Location $sourceRoot + try { + & .\eng\setup-pr-validation.ps1 -prNumber $env:PR_NUMBER -headSha $env:HEAD_SHA -baseSha $env:BASE_SHA + } + finally { + Pop-Location + } + environmentVariables: + PIPELINE_SOURCE_SHA: $(Build.SourceVersion) + PR_NUMBER: ${{ parameters.prNumber }} + HEAD_SHA: ${{ parameters.headSha }} + BASE_SHA: ${{ parameters.baseSha }} + timeout: 0:0:10:0 diff --git a/eng/setup-pr-validation.ps1 b/eng/setup-pr-validation.ps1 new file mode 100644 index 00000000000..75276e9c54f --- /dev/null +++ b/eng/setup-pr-validation.ps1 @@ -0,0 +1,57 @@ +[CmdletBinding(PositionalBinding=$false)] +param ( + [Parameter(Mandatory=$true)] + [ValidatePattern('^[1-9][0-9]*$')] + [string]$prNumber, + [Parameter(Mandatory=$true)] + [ValidatePattern('^[0-9a-f]{40}$')] + [string]$headSha, + [Parameter(Mandatory=$true)] + [ValidatePattern('^[0-9a-f]{40}$')] + [string]$baseSha +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +function Invoke-Git { + # Windows PowerShell treats native stderr (including successful fetch progress) as errors. + $ErrorActionPreference = 'Continue' + $output = & git @args 2>&1 + if ($LASTEXITCODE -ne 0) { + throw "git $args failed: $output" + } + $output | ForEach-Object { "$_" } +} + +if ((Invoke-Git rev-parse HEAD) -ne $headSha) { + throw 'Expected the reviewed PR checkout at the captured head SHA.' +} + +$repository = 'https://github.com/dotnet/fsharp.git' +Invoke-Git fetch --no-tags --depth=2 $repository "refs/pull/$prNumber/merge" | Out-Null +$mergeSha = Invoke-Git rev-parse FETCH_HEAD +$parents = (Invoke-Git show -s --format=%P $mergeSha) -split ' ' +if ($parents.Count -ne 2 -or $parents[0] -ne $baseSha -or $parents[1] -ne $headSha) { + throw 'The fetched PR merge does not match the captured base/head. Request a new /dart or /pr-val run.' +} + +$pr = Invoke-RestMethod -Uri "https://api.github.com/repos/dotnet/fsharp/pulls/$prNumber" -Headers @{ + Accept = 'application/vnd.github+json' + 'User-Agent' = 'FSharp-Apex-Validation' + 'X-GitHub-Api-Version' = '2022-11-28' +} +if ($pr.state -ne 'open' -or $pr.base.repo.full_name -ne 'dotnet/fsharp' -or + $pr.base.ref -ne 'main' -or $pr.head.sha -ne $headSha -or $pr.base.sha -ne $baseSha) { + throw 'The PR is closed or its head/base changed. Request a new /dart or /pr-val run.' +} +$refs = Invoke-Git ls-remote $repository refs/heads/main "refs/pull/$prNumber/head" +foreach ($expected in @("$baseSha`trefs/heads/main", "$headSha`trefs/pull/$prNumber/head")) { + if ($refs -cnotcontains $expected) { + throw 'The current head/base refs do not match the snapshot. Request a new /dart or /pr-val run.' + } +} + +Invoke-Git checkout --detach $mergeSha | Out-Null +Write-Host "PR #$prNumber`: head=$headSha base=$baseSha merge=$mergeSha" +Write-Host "##vso[task.setvariable variable=FSharp.PrMergeSha]$mergeSha" diff --git a/eng/tests/SetupPrValidation.Tests.ps1 b/eng/tests/SetupPrValidation.Tests.ps1 new file mode 100644 index 00000000000..4e0a15924f7 --- /dev/null +++ b/eng/tests/SetupPrValidation.Tests.ps1 @@ -0,0 +1,112 @@ +$ErrorActionPreference = 'Stop' +$setup = Join-Path $PSScriptRoot '..\setup-pr-validation.ps1' +$root = Join-Path ([IO.Path]::GetTempPath()) ("fsharp-pr-validation-" + [Guid]::NewGuid()) +$git = (Get-Command git -CommandType Application).Source +$previousGlobalConfig = $env:GIT_CONFIG_GLOBAL +$previousSystemConfig = $env:GIT_CONFIG_NOSYSTEM + +function Invoke-FixtureGit { + $output = & $git @args + if ($LASTEXITCODE -ne 0) { throw "Fixture git command failed: $args" } + $output +} + +function Invoke-RestMethod { + param($Uri, $Headers) + if ($Uri -ne 'https://api.github.com/repos/dotnet/fsharp/pulls/1') { throw "Unexpected URL: $Uri" } + if ($case -eq 'api-failure') { throw 'GitHub unavailable' } + $pr +} + +New-Item -ItemType Directory $root | Out-Null +Push-Location $root +try { + $env:GIT_CONFIG_GLOBAL = Join-Path $root 'gitconfig' + $env:GIT_CONFIG_NOSYSTEM = '1' + Invoke-FixtureGit init -q -b main remote + Set-Location remote + Invoke-FixtureGit config user.name Test + Invoke-FixtureGit config user.email test@example.invalid + Invoke-FixtureGit commit -q --allow-empty -m base + $base = Invoke-FixtureGit rev-parse HEAD + Invoke-FixtureGit checkout -q -b feature + Invoke-FixtureGit commit -q --allow-empty -m head + $head = Invoke-FixtureGit rev-parse HEAD + Invoke-FixtureGit checkout -q main + Invoke-FixtureGit merge -q --no-ff feature -m merge + $merge = Invoke-FixtureGit rev-parse HEAD + Invoke-FixtureGit commit -q --allow-empty -m later + $later = Invoke-FixtureGit rev-parse HEAD + $remote = (Get-Location).Path.Replace('\', '/') + Set-Location $root + + $cases = @{ + valid = $null + 'head-ref-changed' = 'current head/base refs' + 'base-ref-changed' = 'current head/base refs' + 'merge-mismatch' = 'fetched PR merge' + 'missing-merge' = 'git fetch' + closed = 'PR is closed' + 'non-main' = 'PR is closed' + 'api-head-changed' = 'PR is closed' + 'api-base-changed' = 'PR is closed' + 'api-failure' = 'GitHub unavailable' + 'wrong-reviewed-checkout' = 'reviewed PR checkout' + 'invalid-input' = 'headSha' + } + foreach ($case in $cases.Keys) { + Invoke-FixtureGit -C $remote update-ref refs/heads/main $base + Invoke-FixtureGit -C $remote update-ref refs/pull/1/head $head + Invoke-FixtureGit -C $remote update-ref refs/pull/1/merge $merge + Invoke-FixtureGit clone -q --no-local --depth=1 $remote $case + Push-Location $case + try { + Invoke-FixtureGit config "url.$remote.insteadOf" 'https://github.com/dotnet/fsharp.git' + Invoke-FixtureGit fetch -q --depth=1 $remote refs/pull/1/head + Invoke-FixtureGit checkout -q --detach $head + $pr = @{ + state = 'open' + head = @{ sha = $head } + base = @{ sha = $base; ref = 'main'; repo = @{ full_name = 'dotnet/fsharp' } } + } + $expectedHead = $head + switch ($case) { + 'head-ref-changed' { Invoke-FixtureGit -C $remote update-ref refs/pull/1/head $later } + 'base-ref-changed' { Invoke-FixtureGit -C $remote update-ref refs/heads/main $later } + 'merge-mismatch' { Invoke-FixtureGit -C $remote update-ref refs/pull/1/merge $later } + 'missing-merge' { Invoke-FixtureGit -C $remote update-ref -d refs/pull/1/merge } + 'closed' { $pr.state = 'closed' } + 'non-main' { $pr.base.ref = 'release/test' } + 'api-head-changed' { $pr.head.sha = $later } + 'api-base-changed' { $pr.base.sha = $later } + 'wrong-reviewed-checkout' { + Invoke-FixtureGit fetch -q --depth=1 $remote refs/heads/main + Invoke-FixtureGit checkout -q --detach $base + } + 'invalid-input' { $expectedHead = 'not-a-sha' } + } + $before = Invoke-FixtureGit rev-parse HEAD + $failure = $null + try { & $setup -prNumber 1 -headSha $expectedHead -baseSha $base } + catch { $failure = "$_" } + if ($null -eq $cases[$case]) { + if ($failure) { throw "$case failed: $failure" } + if ((Invoke-FixtureGit rev-parse HEAD) -ne $merge) { throw 'Did not check out the verified merge' } + if ((& $git config --get-regexp 'http\..*extraheader')) { throw 'Persisted checkout credentials' } + } + else { + if (!$failure -or $failure -notmatch $cases[$case]) { throw "$case failed for wrong reason: $failure" } + if ((Invoke-FixtureGit rev-parse HEAD) -ne $before) { throw "$case changed HEAD before validation" } + } + Write-Host "PASS: $case" + } + finally { Pop-Location } + } +} +finally { + Pop-Location + $env:GIT_CONFIG_GLOBAL = $previousGlobalConfig + $env:GIT_CONFIG_NOSYSTEM = $previousSystemConfig + Remove-Item -LiteralPath $root -Recurse -Force +} +exit 0 diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs new file mode 100644 index 00000000000..453cac96bb6 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/Build/BuildProjectTests.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/BuildProjectTests.cs. The project is created from +// the .NET SDK class-library template (via `dotnet new`, matching the source tests; Apex's +// ProjectTemplate.ClassLibrary is the legacy .NET Framework template). The template auto-opens the +// default source file, so the source is set through the open document and saved to disk (an +// out-of-band disk write would be superseded by the still-open template buffer, and the compiler would +// build the valid template instead of the intended code). Rather than parsing the "Build: 1 succeeded, +// 0 failed ..." output-pane summary string, these assert on the idiomatic Apex build result: +// BuildManager.Succeeded and BuildManager.Verify.HasFailedWithErrors for the error case. + +using System.IO; +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class BuildProjectTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify a well-formed F# project builds successfully.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void SuccessfulBuild() + { + var code = @"module Test + +let answer = 42"; + + this.SetLibraryContent(code); + + Assert.IsTrue(this.BuildSolutionSucceeded(), "Build was expected to succeed."); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify an F# project with an incomplete binding fails to build with FS0010.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void FailedBuild() + { + var code = @"module Test + +let answer ="; + + this.SetLibraryContent(code); + + this.BuildSolution(); + + var buildManager = this.Library.VisualStudio.ObjectModel.Solution.BuildManager; + Assert.IsFalse(buildManager.Succeeded, "Build was expected to fail."); + + // The Apex error-list verifier matches the error DESCRIPTION (the message text); the code + // 'FS0010' lives in a separate Code column, so match on the FS0010 message instead. + Assert.IsTrue( + buildManager.Verify.HasFailedWithErrors( + new[] { "Incomplete structured construct" }, alsoVerifyInOutputWindow: false), + "Build was expected to fail with the FS0010 'Incomplete structured construct' error."); + } + + /// + /// Creates an F# class library and replaces its default Library.fs with + /// through the open document (saved to disk), verifying the file on disk matches before building. + /// + private void SetLibraryContent(string code) + { + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Library.fs"); + document.ReplaceAllAndSave(code); + + AssertSourceEquals(code, File.ReadAllText(document.FilePath)); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs new file mode 100644 index 00000000000..515cd14883f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/CodeActionTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/CodeActionTests.cs. Each case sets up the same +// triggering F# source, places the caret where that test does, and verifies the same code/error fix +// is offered on the light bulb (asserting on the action's display text). The Apex light-bulb model +// exposes a flat action list, so the CodeFix/ErrorFix category asserted by the source tests is not +// mirrored here. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class CodeActionTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Remove unused open declarations' code fix is offered on an unused open.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void UnusedOpenDeclarations() + { + var code = @"module Library + +open System + +let x = 42"; + + this.AssertCodeActionOffered(code, "open System", "Remove unused open declarations"); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Add missing 'fun' keyword' error fix is offered on a lambda missing 'fun'.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void AddMissingFunKeyword() + { + var code = @"module Library + +let original = [] +let transformed = original |> List.map (x -> x)"; + + this.AssertCodeActionOffered(code, "->", "Add missing 'fun' keyword"); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the 'Add 'new' keyword' code fix is offered when constructing a disposable.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void AddNewKeywordToDisposables() + { + var code = @"module Library + +let sr = System.IO.StreamReader("""")"; + + this.AssertCodeActionOffered(code, "let sr", "Add 'new' keyword"); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs new file mode 100644 index 00000000000..1aa66153b95 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CodeFixes/SuggestionCodeFix.cs @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Ported verbatim from the TypeScript-VS repository +// (VS/LanguageService/Tests/Integration/CodeFixes/SuggestionCodeFix.cs) and then adjusted so that +// the setup is tailored for the F# extension. A single test case is kept, exercising an F# code fix +// (Wrap expression in parentheses, FS0597). The triggering code is taken from the F# code-fix unit +// tests: vsintegration/tests/FSharp.Editor.Tests/CodeFixes/WrapExpressionInParenthesesTests.fs. + +using System; +using System.Linq; +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class SuggestionCodeFix : FSharpLanguageServiceApexTest + { + #region Private members + protected override bool AutomaticallyDismissMessageBoxes => false; + private const string code = @"module Library + +let rng = System.Random() + +printfn ""Hello %d"" rng.Next(5)"; + private const string expectedCode = @"module Library + +let rng = System.Random() + +printfn ""Hello %d"" (rng.Next(5))"; + #endregion + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify code-fix offers suggestion to wrap an expression in parentheses in an F# file.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void SuggestionCodeFixInFSharpFile() + { + var document = this.OpenFSharpDocument(code, "rng.Next"); + + this.InvokeCodeFix(document, "rng.Next"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + // Poll until the fix rewrites the buffer, then assert it actually applied. Without the + // assertion a fix that never fires would leave the original source and the test would still + // pass. Compare on normalized source so incidental line-ending differences do not race the poll. + this.Library.Synchronization.TryWaitForCondition( + () => NormalizeSource(document.Contents) == NormalizeSource(expectedCode), + TimeSpan.FromSeconds(15)); + + AssertSourceEquals(expectedCode, document.Contents); + } + + #region Helps + public void InvokeCodeFix(TextDocumentView document, string expression) + { + var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, expression); + var globalScopeAction = lightBulb.Actions.Single(action => + action.Text.IndexOf("Wrap", StringComparison.OrdinalIgnoreCase) >= 0 + ); + + globalScopeAction.Invoke(); + } + #endregion + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs new file mode 100644 index 00000000000..4ce393246a3 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/CreateProject/CreateProjectTests.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/CreateProjectTests.cs. Each case creates an F# +// project from a .NET SDK template (via `dotnet new`, matching the SDK templates the source tests +// target — Apex's ProjectTemplate enum maps to the legacy .NET Framework F# templates with different +// filenames and content) and asserts the auto-generated default source. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class CreateProjectTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# class library project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void ClassLibrary() + { + var expectedCode = @"namespace Library + +module Say = + let hello name = + printfn ""Hello %s"" name"; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("classlib", "Library"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Library.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# console application project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void ConsoleApp() + { + var expectedCode = @"// For more information see https://aka.ms/fsharp-console-apps +printfn ""Hello from F#"""; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("console", "ConsoleApp"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Program.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify the default source of a new F# xUnit test project.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void XUnitTestProject() + { + var expectedCode = @"module Tests + +open System +open Xunit + +[] +let ``My test`` () = + Assert.True(true)"; + + var project = this.Library.ProjectCreation.CreateFSharpProjectFromSdkTemplate("xunit", "Tests"); + this.Library.Synchronization.WaitForSolutionCrawler(); + + var document = this.Library.OpenProjectFile(project, "Tests.fs"); + + AssertSourceEquals(expectedCode, document.Contents); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj new file mode 100644 index 00000000000..6750b2a6a34 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/FSharp.Editor.Apex.IntegrationTests.csproj @@ -0,0 +1,41 @@ + + + + + net472 + preview + disable + Library + false + + true + $(NoWarn);CS1591;VSTHRD200 + + + + + + + + + + + + + + + + diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs new file mode 100644 index 00000000000..5eff72aa107 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/GoToDefinition/GoToDefinitionTests.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. +// +// Apex counterparts of the xUnit/IdeFact tests in +// vsintegration/tests/FSharp.Editor.IntegrationTests/GoToDefinitionTests.cs. Go To Definition is +// invoked idiomatically via the editor caret (IVisualStudioCaretTestExtension.GoToDefinition), and the +// result is read from the active document (current line + window caption) rather than the async +// SolutionExplorer/Editor/Shell helpers used by the xUnit harness. + +using FSharp.Editor.Apex.IntegrationTests.TestFramework; +using Microsoft.Test.Apex.VisualStudio.Solution; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests +{ + [TestClass] + public class GoToDefinitionTests : FSharpLanguageServiceApexTest + { + protected override bool AutomaticallyDismissMessageBoxes => false; + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify Go To Definition navigates from a use of a binding to its definition.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void GoesToDefinition() + { + var code = @"module Test + +let add x y = x + y + +let increment = add 1"; + + var document = this.OpenFSharpDocument(code, "add 1"); + + // Resolve references so the file type-checks and Go To Definition can bind the symbol. + this.BuildSolution(); + + this.GoToDefinitionAndWait(document, "add 1", line => line.Contains("let add x y = x + y")); + } + + [RetryTestMethod] + [Timeout(ShortTimeoutMS)] + [Description("Verify Go To Definition stays within the signature file and the implementation file respectively.")] + [Owner("fsharptools")] + [TestCategory("nightly"), TestCategory("Walkthrough")] + public void FsiAndFsFilesGoToCorrespondentDefinitions() + { + var fsi = @"module Module + +type SomeType = +| Number of int +| Letter of char + +val id: t: SomeType -> SomeType"; + + var fs = @"module Module + +type SomeType = + | Number of int + | Letter of char + +let id (t: SomeType) = t"; + + var project = this.Library.ProjectCreation.CreateFSharpProject(ProjectTemplate.ClassLibrary, "Library"); + + // The signature file must precede the implementation file in F# compile order. When a file is + // added, VS inserts it alphabetically, so adding "Module.fsi" then "Module.fs" would place the + // implementation first ("Module.fs" sorts before "Module.fsi") and the signature would not + // apply. Add the signature under a name that sorts first, then rename it — the same approach as + // the source IdeFact test. + var fsiItem = this.Library.ProjectCreation.AddProjectItemFromContent(project, "AModule.fsi", fsi); + this.Library.ProjectCreation.AddProjectItemFromContent(project, "Module.fs", fs); + fsiItem.Rename("Module.fsi"); + this.Library.Synchronization.WaitForSolutionCrawler(); + this.BuildSolution(); + + // From the signature file, Go To Definition on the type usage stays in Module.fsi. + var fsiDocument = this.Library.OpenProjectFile(project, "Module.fsi"); + this.GoToDefinitionAndWait(fsiDocument, "SomeType ->", line => line.Trim() == "type SomeType =", "Module.fsi"); + + // From the implementation file, Go To Definition on the type usage stays in Module.fs. + var fsDocument = this.Library.OpenProjectFile(project, "Module.fs"); + this.GoToDefinitionAndWait(fsDocument, "SomeType)", line => line.Trim() == "type SomeType =", "Module.fs"); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs new file mode 100644 index 00000000000..4583a7b321f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceApexTest.cs @@ -0,0 +1,317 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Web.Script.Serialization; +using Microsoft.Test.Apex.Editor; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Base test class for F# language service tests built using the Apex framework. + /// Compact, self-contained equivalent of the TypeScript-VS LanguageServiceApexTest, tailored for + /// the F# extension and without the JavaScript/TypeScript BrowserTools layer. + /// + [TestClass] + public abstract class FSharpLanguageServiceApexTest : VisualStudioHostTest + { + protected const int ShortTimeoutMS = 1000 * 60 * 12; + protected const int LongTimeoutMS = 1000 * 60 * 20; + + // Apex launches the Visual Studio instance pointed to by this environment variable, if set. + private const string InstallationUnderTestPathVariable = "VisualStudio.InstallationUnderTest.Path"; + + private FSharpLanguageServiceLibrary libraryCache; + + /// + /// Gets the F# language service test library. + /// + public FSharpLanguageServiceLibrary Library + => this.libraryCache ??= new FSharpLanguageServiceLibrary(this.VisualStudio, this.Operations); + + /// + /// Whether message boxes shown during the test run should be dismissed automatically. + /// + protected virtual bool AutomaticallyDismissMessageBoxes => true; + + /// + /// The registry root suffix (experimental hive) of the Visual Studio instance to launch. + /// Defaults to "RoslynDev", where the locally-built F# extension is deployed. + /// + protected virtual string RootSuffix => "RoslynDev"; + + /// + /// The product milestone of the installed Visual Studio to launch, as reported by vswhere's + /// catalog.productMilestone. Defaults to "Canary" (the IntCanary channel), which selects the + /// Canary install rather than Insiders when both are present locally. On CI the installed VS is + /// a different channel (e.g. "Preview"), so this can be overridden with the + /// FSHARP_APEX_VS_MILESTONE environment variable; setting VisualStudio.InstallationUnderTest.Path + /// directly takes precedence over milestone resolution entirely. + /// + protected virtual string TargetProductMilestone + => Environment.GetEnvironmentVariable("FSHARP_APEX_VS_MILESTONE") is string milestone + && !string.IsNullOrEmpty(milestone) + ? milestone + : "Canary"; + + protected override VisualStudioHostConfiguration GetVisualStudioHostConfiguration() + { + this.EnsureTargetInstallationSelected(); + + var config = base.GetVisualStudioHostConfiguration(); + config.AutomaticallyDismissMessageBoxes = this.AutomaticallyDismissMessageBoxes; + config.RootSuffix = this.RootSuffix; + + // On a fresh experimental hive (e.g. CI), devenv shows a full-screen "Sign in to Visual Studio" + // first-launch dialog that blocks the UI thread, so it never registers its DTE automation + // object and Apex times out waiting for it. The devenv '/NoSigninPrompt' switch (registered by + // Microsoft.VisualStudio.Shell.Connected under AppCommandLine) suppresses that dialog. + config.CommandLineArguments = + string.IsNullOrEmpty(config.CommandLineArguments) + ? "/NoSigninPrompt" + : config.CommandLineArguments + " /NoSigninPrompt"; + + return config; + } + + /// + /// Points Apex at the Visual Studio install matching + /// (e.g. Canary). An explicit VisualStudio.InstallationUnderTest.Path override is respected. + /// + private void EnsureTargetInstallationSelected() + { + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable(InstallationUnderTestPathVariable))) + { + return; + } + + string devenvPath = ResolveDevenvByMilestone(this.TargetProductMilestone); + if (!string.IsNullOrEmpty(devenvPath)) + { + Environment.SetEnvironmentVariable(InstallationUnderTestPathVariable, devenvPath); + } + } + + private static string ResolveDevenvByMilestone(string milestone) + { + string programFilesX86 = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86); + string vswhere = Path.Combine(programFilesX86, "Microsoft Visual Studio", "Installer", "vswhere.exe"); + if (!File.Exists(vswhere)) + { + return null; + } + + var startInfo = new ProcessStartInfo(vswhere, "-all -prerelease -format json -utf8") + { + RedirectStandardOutput = true, + UseShellExecute = false, + CreateNoWindow = true, + }; + + string json; + using (var process = Process.Start(startInfo)) + { + json = process.StandardOutput.ReadToEnd(); + process.WaitForExit(); + } + + if (string.IsNullOrWhiteSpace(json) + || !(new JavaScriptSerializer { MaxJsonLength = int.MaxValue }.DeserializeObject(json) is object[] installations)) + { + return null; + } + + foreach (var installation in installations.OfType>()) + { + if (installation.TryGetValue("catalog", out var catalogObject) + && catalogObject is Dictionary catalog + && catalog.TryGetValue("productMilestone", out var installedMilestone) + && string.Equals(installedMilestone as string, milestone, StringComparison.OrdinalIgnoreCase) + && installation.TryGetValue("productPath", out var productPath)) + { + return productPath as string; + } + } + + return null; + } + + /// + /// Moves the caret onto , triggers the code-fix request and + /// expands the resulting light bulb. + /// + protected ILightBulbTestExtension MoveToExpressionAndExpandLightBulb(TextDocumentView document, string expression) + { + // Nudging the caret into the diagnostic range triggers a request for code fixes. + document.MoveToExpression(expression); + document.MoveRight(); + document.MoveLeft(); + + this.Library.Synchronization.WaitForLightBulb(); + + Assert.IsTrue( + this.Library.Editor.LightBulb.Verify.IsLightBulbPresent(TimeSpan.FromSeconds(60)), + $"Expected a light bulb to be present at expression '{expression}'."); + + var lightBulb = this.Library.Editor.LightBulb.GetActiveLightBulb(); + lightBulb.Expand(); + + // Wait for the suggested actions to populate. Poll the already-expanded session directly: + // re-querying GetActiveLightBulb() here returns null once the light bulb is expanded into + // its flyout (LightBulbBroker no longer reports it as the active session), and the flyout + // taking focus can also null out Library.Editor (ActiveDocumentWindowAsTextEditor) — either + // of which previously threw a NullReferenceException inside the poll. LightBulbTestExtension + // .Actions reads the captured session and returns an empty (never null) set while pending. + this.Library.Synchronization.TryWaitForCondition(() => lightBulb.Actions.Any()); + + return lightBulb; + } + + /// + /// Creates a fresh single-project F# solution whose Library.fs contains + /// and opens it in the editor. Waits for the project to finish loading before editing (so the + /// F# project system does not reload the empty file over the insert) and confirms + /// reached the buffer before returning, so downstream caret + /// searches are not racing document initialization. + /// + protected TextDocumentView OpenFSharpDocument(string code, string expectedContent) + { + var project = this.Library.ProjectCreation.CreateFSharpLibrary(); + var documentItem = this.Library.ProjectCreation.AddProjectItemFromEmptyFile(project, "Library.fs"); + var document = this.Library.OpenDocument(documentItem); + + this.Library.Synchronization.WaitForSolutionCrawler(); + + document.InsertText(code); + + Assert.IsTrue( + this.Library.Synchronization.TryWaitForCondition( + () => document.Contents.Contains(expectedContent), TimeSpan.FromSeconds(15)), + $"Inserted source did not appear in the document buffer (looking for '{expectedContent}')."); + + return document; + } + + /// + /// Builds the current solution and waits for it to finish. Building performs a NuGet restore and + /// resolves references — which the F# language service needs before it can type-check and surface + /// SEMANTIC diagnostics (e.g. the unused-open analyzer, or FS0760 for disposables). This is the + /// Apex counterpart of the source IdeFact tests' RestoreNuGetPackagesAsync + WaitForProjectSystem. + /// Parse-based fixes (FS0597, FS0010) don't need it. Build success is not asserted: some fixture + /// sources intentionally contain errors, and a failed compile still restores references. + /// + protected void BuildSolution() + { + this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Build(waitForBuildToFinish: true); + this.Library.Synchronization.WaitForSolutionCrawler(); + } + + /// + /// Builds the current solution and returns whether it succeeded. + /// + protected bool BuildSolutionSucceeded() + { + this.BuildSolution(); + return this.Library.VisualStudio.ObjectModel.Solution.BuildManager.Succeeded; + } + + /// + /// Places the caret on , focuses the editor and invokes Go To + /// Definition, then waits for navigation to settle. The result is read from the active document + /// afterwards (which may be the same or a different file). + /// + protected void GoToDefinition(TextDocumentView document, string expression) + { + document.MoveToExpression(expression); + document.Focus(); + document.GoToDefinition(); + this.Library.Synchronization.WaitForSolutionCrawler(); + } + + /// + /// Invokes Go To Definition on and polls until the active document + /// settles on a line matching and, when given, a window caption + /// equal to . Go To Definition navigates asynchronously (it may + /// open/activate another document and only then move the caret), so — exactly like the light-bulb + /// helper waits for its actions — we poll for the end state instead of reading it immediately. + /// Reading it immediately is the fire-and-assert race that makes the ported navigation tests flaky. + /// + protected void GoToDefinitionAndWait( + TextDocumentView document, + string expression, + Func lineMatches, + string expectedCaption = null) + { + this.GoToDefinition(document, expression); + + string lastLine = null; + string lastCaption = null; + bool landed = this.Library.Synchronization.TryWaitForCondition( + () => + { + lastLine = this.Library.ActiveDocumentCurrentLineText; + lastCaption = this.Library.ActiveDocumentCaption; + return lastLine != null + && lineMatches(lastLine) + && (expectedCaption == null || string.Equals(lastCaption, expectedCaption, StringComparison.Ordinal)); + }, + TimeSpan.FromSeconds(30)); + + Assert.IsTrue( + landed, + $"Go To Definition on '{expression}' did not settle on the expected location. " + + (expectedCaption != null ? $"Expected caption '{expectedCaption}', actual '{lastCaption}'. " : string.Empty) + + $"Actual current line: '{lastLine}'."); + } + + /// + /// Asserts two pieces of F# source are equal, ignoring line-ending style and any trailing + /// newline. Template output and editor buffers differ in those incidental ways across SDKs, so + /// normalizing avoids brittle failures while still comparing the meaningful content exactly. + /// + protected static void AssertSourceEquals(string expected, string actual) + { + Assert.AreEqual(NormalizeSource(expected), NormalizeSource(actual)); + } + + /// + /// Normalizes F# source for comparison by unifying line-ending style and trimming any trailing + /// newline, so comparisons ignore those incidental differences. + /// + protected static string NormalizeSource(string source) + => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n'); + + /// + /// Verifies that placing the caret on in the given F# source + /// offers exactly one light-bulb action whose text contains . + /// Mirrors the display-text assertions in FSharp.Editor.IntegrationTests CodeActionTests (the Apex + /// light-bulb model exposes a flat action list, so the code-fix/error-fix category is not checked). + /// + protected void AssertCodeActionOffered(string code, string caretExpression, string expectedActionText) + { + var document = this.OpenFSharpDocument(code, caretExpression); + + // Restore/resolve references so the F# language service can type-check and offer semantic + // fixes (the unused-open analyzer, FS0760, ...); without this their light bulb never appears. + this.BuildSolution(); + + var lightBulb = this.MoveToExpressionAndExpandLightBulb(document, caretExpression); + + var matchCount = lightBulb.Actions.Count(action => + action.Text.IndexOf(expectedActionText, StringComparison.OrdinalIgnoreCase) >= 0); + + Assert.AreEqual( + 1, + matchCount, + $"Expected exactly one code action containing '{expectedActionText}'. Offered actions: " + + string.Join(", ", lightBulb.Actions.Select(action => $"'{action.Text}'"))); + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs new file mode 100644 index 00000000000..a3b98ab410f --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/FSharpLanguageServiceLibrary.cs @@ -0,0 +1,82 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Linq; +using Microsoft.Test.Apex; +using Microsoft.Test.Apex.Services; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.Test.Apex.VisualStudio.Editor; +using Microsoft.Test.Apex.VisualStudio.Shell; +using Microsoft.Test.Apex.VisualStudio.Solution; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Compact test library for the F# Apex integration tests. Provides the small facade over the + /// Apex that the code-action tests use (project creation, + /// synchronization, the active editor and document opening), tailored for the F# extension. + /// + public sealed class FSharpLanguageServiceLibrary + { + public FSharpLanguageServiceLibrary(VisualStudioHost visualStudio, IOperations operations) + { + this.VisualStudio = visualStudio; + this.Synchronization = new SynchronizationHelper(visualStudio, operations.Get()); + this.ProjectCreation = new ProjectCreationHelper(visualStudio); + } + + /// The Apex Visual Studio host. + public VisualStudioHost VisualStudio { get; } + + /// Synchronization helpers used to wait for background work. + public SynchronizationHelper Synchronization { get; } + + /// F# project and project-item creation helpers. + public ProjectCreationHelper ProjectCreation { get; } + + /// The editor of the active document window, or null if none is active. + public IVisualStudioTextEditorTestExtension Editor + => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindowAsTextEditor?.Editor; + + /// + /// Opens the given project item in the text editor and returns a view over it. + /// + public TextDocumentView OpenDocument(ProjectItemTestExtension documentItem) + { + var window = documentItem.Open(); + return new TextDocumentView(window); + } + + /// + /// Finds under , opens it in the text + /// editor and returns a view over it. The project's item tree is populated lazily, so the lookup + /// is polled (TryFindChild is single-shot); on timeout it throws listing the project's actual + /// top-level item names, which makes a template-filename surprise obvious. + /// + public TextDocumentView OpenProjectFile(ProjectTestExtension project, string fileName) + { + ProjectItemTestExtension item = null; + this.Synchronization.TryWaitForCondition( + () => (item = project.TryFindChild(fileName, true)) != null); + + if (item == null) + { + var actualNames = string.Join(", ", project.ProjectItems.Select(child => $"'{child.Name}'")); + throw new InvalidOperationException( + $"Project item '{fileName}' was not found in project '{project.Name}'. Items present: {actualNames}."); + } + + return this.OpenDocument(item); + } + + /// The short caption (file name) of the active document window, or null if none is active. + public string ActiveDocumentCaption + => this.VisualStudio.ObjectModel.WindowManager.ActiveDocumentWindow?.Caption; + + /// The text of the caret's current line in the active document editor, or null if none is active. + public string ActiveDocumentCurrentLineText + => this.Editor?.Caret.GetCurrentLineText(); + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs new file mode 100644 index 00000000000..11a837bd233 --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/ProjectCreationHelper.cs @@ -0,0 +1,117 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Diagnostics; +using System.IO; +using System.Threading.Tasks; +using Microsoft.Test.Apex.VisualStudio; +using Microsoft.Test.Apex.VisualStudio.Solution; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Creates F# projects and project items for the Apex integration tests. + /// Tailored for the F# extension: the TypeScript-VS ProjectCreationHelper created ASP.NET Core + /// web apps for JavaScript/TypeScript; this one creates plain F# class libraries. + /// + public sealed class ProjectCreationHelper + { + private readonly VisualStudioHost visualStudio; + private int projectCounter = 0; + + public ProjectCreationHelper(VisualStudioHost visualStudio) + { + this.visualStudio = visualStudio; + } + + /// + /// Creates a new F# class library in a fresh solution. + /// + public ProjectTestExtension CreateFSharpLibrary() + { + string projectName = $"FSharpLibrary_{++this.projectCounter}"; + return this.CreateFSharpProject(ProjectTemplate.ClassLibrary, projectName); + } + + /// + /// Creates a new F# project of the given built-in template with the given name. + /// + public ProjectTestExtension CreateFSharpProject(ProjectTemplate template, string projectName) + => this.visualStudio.ObjectModel.Solution.CreateProject( + ProjectLanguage.FSharp, + template, + projectName); + + /// + /// Creates a fresh single-project solution from an F# SDK template (e.g. "classlib", "console", + /// "xunit") scaffolded on disk with dotnet new. Apex's enum + /// maps to the legacy .NET Framework F# templates (which produce Library1.fs/Script.fsx), whereas + /// the source FSharp.Editor.IntegrationTests target the .NET SDK templates; using dotnet new + /// makes the project (filenames and default source) match those. An empty solution is created + /// first because AddProject throws when no solution is open. + /// + public ProjectTestExtension CreateFSharpProjectFromSdkTemplate(string sdkTemplateName, string projectName) + { + string projectDirectory = Path.Combine( + Path.GetTempPath(), "FSharpApexTemplates", Guid.NewGuid().ToString("N"), projectName); + Directory.CreateDirectory(projectDirectory); + + RunDotNet($"new {sdkTemplateName} --language \"F#\" --name \"{projectName}\" --output \"{projectDirectory}\""); + + string projectPath = Path.Combine(projectDirectory, projectName + ".fsproj"); + this.visualStudio.ObjectModel.Solution.CreateEmptySolution(); + return this.visualStudio.ObjectModel.Solution.AddProject(projectPath); + } + + /// + /// Creates an empty file on disk and adds it to the given project or project item. + /// + public ProjectItemTestExtension AddProjectItemFromEmptyFile(IProjectItemsContainer container, string fileName) + { + string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath; + string filePath = Path.Combine(containerDirectory, fileName); + File.WriteAllText(filePath, string.Empty); + return container.AddProjectItemFromFile(filePath); + } + + /// + /// Creates a file with the given content on disk and adds it to the given project or item. + /// + public ProjectItemTestExtension AddProjectItemFromContent(IProjectItemsContainer container, string fileName, string content) + { + string containerDirectory = container.IsProject ? container.ProjectDirectory : container.FullPath; + string filePath = Path.Combine(containerDirectory, fileName); + File.WriteAllText(filePath, content); + return container.AddProjectItemFromFile(filePath); + } + + private static void RunDotNet(string arguments) + { + var startInfo = new ProcessStartInfo("dotnet", arguments) + { + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardOutput = true, + RedirectStandardError = true, + }; + + using var process = Process.Start(startInfo) + ?? throw new InvalidOperationException($"Failed to start 'dotnet {arguments}'."); + + // Drain both redirected streams concurrently. Reading one to completion before the other can + // deadlock: if the child fills the stdout pipe buffer while we block on stderr's EOF (which + // only arrives at process exit), neither side can make progress. + Task standardErrorTask = process.StandardError.ReadToEndAsync(); + process.StandardOutput.ReadToEnd(); + string standardError = standardErrorTask.GetAwaiter().GetResult(); + process.WaitForExit(); + + if (process.ExitCode != 0) + { + throw new InvalidOperationException($"'dotnet {arguments}' failed with exit code {process.ExitCode}: {standardError}"); + } + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs new file mode 100644 index 00000000000..a5f834042ec --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/RetryTestMethodAttribute.cs @@ -0,0 +1,62 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +using System; +using System.Diagnostics; +using System.Linq; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Drop-in replacement for that re-runs a test up to + /// maxAttempts times, passing as soon as one attempt passes. These Apex tests drive a real + /// Visual Studio through UI automation and are occasionally flaky (focus, background analysis and + /// build/restore timing); a bounded retry keeps them reliable without masking a genuine failure — + /// every attempt has to fail before the test is reported as failed. + /// + /// Each attempt goes through the full per-test pipeline (a fresh test-class instance plus + /// TestInitialize/TestCleanup, i.e. a fresh VS session for the Apex host), so a retry starts from a + /// clean state rather than reusing the dirtied state of the previous attempt. + /// + /// MSTest's own [Retry] attribute is only available from 3.8; this repo pins 3.6.4, so the + /// classic "derive from TestMethodAttribute and override Execute" idiom is used instead. + /// + [AttributeUsage(AttributeTargets.Method, AllowMultiple = false)] + public sealed class RetryTestMethodAttribute : TestMethodAttribute + { + private readonly int maxAttempts; + + public RetryTestMethodAttribute(int maxAttempts = 2) + { + if (maxAttempts < 1) + { + throw new ArgumentOutOfRangeException(nameof(maxAttempts), maxAttempts, "At least one attempt is required."); + } + + this.maxAttempts = maxAttempts; + } + + public override TestResult[] Execute(ITestMethod testMethod) + { + TestResult[] results = null; + + for (var attempt = 1; attempt <= this.maxAttempts; attempt++) + { + results = base.Execute(testMethod); + + if (results.All(result => result.Outcome == UnitTestOutcome.Passed)) + { + return results; + } + + if (attempt < this.maxAttempts) + { + Trace.WriteLine( + $"[RetryTestMethod] '{testMethod.TestMethodName}' failed on attempt {attempt} of {this.maxAttempts}; retrying."); + } + } + + return results; + } + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs new file mode 100644 index 00000000000..80969ffd74c --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/SynchronizationHelper.cs @@ -0,0 +1,58 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using Microsoft.Test.Apex.Services; +using Microsoft.Test.Apex.VisualStudio; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Synchronization helper for the F# Apex integration tests. + /// This is a compact, self-contained equivalent of the TypeScript-VS SynchronizationHelper: + /// where that type delegated to the Roslyn "WaitForFeatures" async-operation waiter, this one + /// polls via the Apex so the harness has no dependency on + /// the JavaScript/TypeScript BrowserTools layer. + /// + public sealed class SynchronizationHelper + { + private static readonly TimeSpan DefaultTimeout = TimeSpan.FromSeconds(30); + private static readonly TimeSpan DefaultInterval = TimeSpan.FromSeconds(1); + + private readonly VisualStudioHost visualStudio; + private readonly ISynchronizationService synchronizationService; + + public SynchronizationHelper(VisualStudioHost visualStudio, ISynchronizationService synchronizationService) + { + this.visualStudio = visualStudio; + this.synchronizationService = synchronizationService; + } + + /// + /// Waits for the solution to be fully loaded and for background analysis to settle. + /// + public void WaitForSolutionCrawler() + { + this.visualStudio.ObjectModel.Solution.WaitForFullyLoaded(); + this.Settle(); + } + + /// + /// Waits for the light bulb / code-action pipeline to settle before the light bulb is queried. + /// + public void WaitForLightBulb() => this.Settle(); + + /// + /// Polls until it returns true or the timeout elapses. + /// + /// True if the condition became true within the timeout; otherwise false. + public bool TryWaitForCondition(Func condition, TimeSpan? timeout = null) + { + return this.synchronizationService.TryWaitFor(timeout ?? DefaultTimeout, () => condition()); + } + + private void Settle() + => this.synchronizationService.TryWaitFor(DefaultInterval, () => false); + } +} diff --git a/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs new file mode 100644 index 00000000000..47e756ea87c --- /dev/null +++ b/vsintegration/tests/FSharp.Editor.Apex.IntegrationTests/TestFramework/TextDocumentView.cs @@ -0,0 +1,132 @@ +//----------------------------------------------------------------------------- +// Copyright (c) Microsoft Corporation. All rights reserved. +//----------------------------------------------------------------------------- + +using System; +using System.Text.RegularExpressions; +using System.Threading; +using Microsoft.Test.Apex.VisualStudio.Editor; +using Microsoft.Test.Apex.VisualStudio.Shell; + +namespace FSharp.Editor.Apex.IntegrationTests.TestFramework +{ + /// + /// Represents the text view of an open VS document. Compact, self-contained equivalent of the + /// TypeScript-VS TextDocumentView / TextDocumentViewBase, wrapping the Apex text editor extension + /// with the small set of operations the code-action tests need. + /// + public sealed class TextDocumentView + { + private readonly TextEditorDocumentWindowTestExtension window; + + public TextDocumentView(TextEditorDocumentWindowTestExtension window) + { + this.window = window; + } + + private IVisualStudioTextEditorTestExtension editor => this.window.Editor; + + /// The full text content of the document. + public string Contents => this.editor.Contents; + + /// The path of the file backing this document. + public string FilePath => this.window.FilePath; + + /// + /// Inserts text verbatim at the current caret position via a direct text-buffer edit. + /// Unlike InsertTextWithReturn, this does not simulate per-character typing, so the F# + /// editor's brace/quote auto-completion, IntelliSense auto-commit and smart indentation do not + /// fire and rewrite the buffer. That keeps the inserted source exactly as given, which the + /// code-fix tests rely on when locating expressions afterwards. + /// + public void InsertText(string text) => this.editor.Edit.InsertTextInBuffer(text); + + /// + /// Moves the caret to the first (or -th) occurrence of an + /// expression. Apex's caret search treats the argument as a regular expression, but every caller + /// passes a literal source fragment (which may contain regex metacharacters such as ')' in + /// "SomeType)"), so the input is escaped to match it literally. + /// + public void MoveToExpression(string expression, int matchIndex = 1) + => this.editor.Caret.MoveToExpression(Regex.Escape(expression), matchIndex: matchIndex); + + /// Moves the caret one character to the left. + public void MoveLeft() => this.editor.Caret.MoveLeft(); + + /// Moves the caret one character to the right. + public void MoveRight() => this.editor.Caret.MoveRight(); + + /// Gives this document's editor keyboard focus (needed before caret-driven commands). + public void Focus() => this.editor.Focus(); + + /// The text of the line the caret is currently on (untrimmed). + public string CurrentLineText => this.editor.Caret.GetCurrentLineText(); + + /// Invokes Go To Definition from the current caret position. + public void GoToDefinition() => this.editor.Caret.GoToDefinition(); + + /// + /// Replaces the entire document with and saves it to disk. Build tests + /// need this: the template auto-opens the default file, so the source must be set through the + /// open document (buffer) and saved, otherwise an out-of-band disk write is superseded by the + /// still-open (valid template) buffer and the compiler never sees the intended code. + /// + /// The template content loads into the buffer asynchronously after the document opens, so this + /// first waits for that initial content to arrive and then applies the whole-buffer replacement + /// with verify-and-retry: without this a replacement done too early is overwritten by the + /// late-arriving template content, leaving the original source on disk. The selection is deleted + /// before a direct buffer insert (which does not itself replace a selection and, unlike simulated + /// typing, avoids brace/quote auto-completion). + /// + public void ReplaceAllAndSave(string text) + { + this.editor.Focus(); + + if (!WaitUntil(() => !string.IsNullOrEmpty(this.editor.Contents))) + { + throw new InvalidOperationException("Document did not load its initial content before editing."); + } + + var replaced = WaitUntil(() => + { + this.editor.Selection.SelectAll(); + this.editor.Edit.DeleteCharNext(1); + this.editor.Edit.InsertTextInBuffer(text); + return Normalize(this.editor.Contents) == Normalize(text); + }); + + if (!replaced) + { + throw new InvalidOperationException( + $"Failed to set document content. Expected:\n{text}\nActual:\n{this.editor.Contents}"); + } + + this.window.Save(); + + if (!this.window.TryWaitForNotDirty(TimeSpan.FromSeconds(15), TimeSpan.FromMilliseconds(100))) + { + throw new InvalidOperationException("Document remained dirty after saving."); + } + } + + private static bool WaitUntil(Func condition) + { + var deadline = DateTime.UtcNow + TimeSpan.FromSeconds(15); + do + { + if (condition()) + { + return true; + } + + Thread.Sleep(200); + } + while (DateTime.UtcNow < deadline); + + return false; + } + + private static string Normalize(string source) + => (source ?? string.Empty).Replace("\r\n", "\n").Replace("\r", "\n").TrimEnd('\n'); + } +}