Run Apex VS integration tests on maintainer-requested PRs

.github/scripts/pr-validation.Tests.ps1

@@ -0,0 +1,183 @@
+$ErrorActionPreference = 'Stop'
+$root = Join-Path $PSScriptRoot ('.pr-validation-tests-' + [Guid]::NewGuid())
+$count = 0
+New-Item -ItemType Directory $root | Out-Null
+
+function Assert($Condition, $Message) {
+    if (!$Condition) { throw $Message }
+}
+
+function Assert-Snapshot($Request) {
+    Assert ($Request.resources.repositories.self.refName -ceq 'refs/heads/main') 'Wrong YAML ref'
+    Assert ($Request.resources.repositories.self.version -ceq ('b' * 40)) 'Wrong YAML revision'
+    Assert ($Request.templateParameters.prNumber -is [string] -and $Request.templateParameters.prNumber -ceq '42') 'PR number must be a string'
+    Assert ($Request.templateParameters.headSha -ceq ('a' * 40)) 'Wrong head snapshot'
+    Assert ($Request.templateParameters.baseSha -ceq ('b' * 40)) 'Wrong base snapshot'
+}
+
+function Invoke-WebRequest {
+    [CmdletBinding()]
+    param($Uri, $Method, $Headers, $UserAgent, $ContentType, $MaximumRedirection, [switch]$SkipHttpErrorCheck, $Body)
+    $calls.Add($PSBoundParameters)
+    Assert ($responses.ContainsKey($Uri)) "Unexpected URL: $Uri"
+    Assert ($Headers.Authorization -ceq "Bearer $expectedToken") 'Wrong bearer token'
+    Assert ($Headers.Accept -ceq 'application/json' -and $ContentType -ceq 'application/json') 'Wrong JSON headers'
+    Assert ($UserAgent -ceq 'fsharp-pr-validation') 'Missing user agent'
+    Assert ($MaximumRedirection -ceq 0 -and $SkipHttpErrorCheck -and $PSBoundParameters.ErrorAction -eq 'Stop') 'HTTP handling is not fail-closed'
+    $response = $responses[$Uri]
+    Assert ($Method -ceq $response.Method) 'Wrong HTTP method'
+    if ($response.Failure) { throw 'Network failed (test-token)' }
+    @{
+        StatusCode = $response.Status
+        Content = if ($response.ContainsKey('Raw')) { $response.Raw } else { ConvertTo-Json -InputObject $response.Body -Depth 10 -Compress }
+    }
+}
+
+function Test-Case($Name, [scriptblock]$Change = {}, $ExpectedError = '', $Mode = 'Prepare', $CallCount = -1) {
+    $event = @{
+        comment = @{ body = '/dart'; user = @{ login = 'maintainer'; type = 'User' } }
+        issue = @{ number = 42; pull_request = @{} }
+    }
+    $pr = @{
+        state = 'open'
+        head = @{ sha = 'a' * 40; repo = @{ full_name = 'contributor/fsharp' } }
+        base = @{ sha = 'b' * 40; ref = 'main'; repo = @{ full_name = 'dotnet/fsharp' } }
+    }
+    $permission = @{ permission = 'write' }
+    $request = @{
+        resources = @{ repositories = @{ self = @{ refName = 'refs/heads/main'; version = 'b' * 40 } } }
+        templateParameters = @{ prNumber = '42'; headSha = 'a' * 40; baseSha = 'b' * 40 }
+    }
+    $permissionUri = 'https://api.github.com/repos/dotnet/fsharp/collaborators/maintainer/permission'
+    $pullUri = 'https://api.github.com/repos/dotnet/fsharp/pulls/42'
+    $memberUri = 'https://api.github.com/orgs/microsoft/members/maintainer'
+    $queueUri = 'https://dev.azure.com/devdiv/DevDiv/_apis/pipelines/123/runs?api-version=7.1'
+    $reportUri = 'https://api.github.com/repos/dotnet/fsharp/issues/42/comments'
+    $responses = @{
+        $permissionUri = @{ Method = 'GET'; Status = 200; Body = $permission }
+        $pullUri = @{ Method = 'GET'; Status = 200; Body = $pr }
+        $memberUri = @{ Method = 'GET'; Status = 204 }
+        $queueUri = @{ Method = 'POST'; Status = 200; Body = @{ id = 456 } }
+        $reportUri = @{ Method = 'POST'; Status = 201; Body = @{ id = 789 } }
+    }
+    $calls = [Collections.Generic.List[object]]::new()
+    $env:GITHUB_EVENT_PATH = Join-Path $root 'event.json'
+    $env:GITHUB_OUTPUT = Join-Path $root 'output'
+    $env:GITHUB_REPOSITORY = 'dotnet/fsharp'; $env:GITHUB_ACTOR = 'rerunning-admin'
+    $env:GITHUB_SERVER_URL = 'https://github.com'; $env:GITHUB_RUN_ID = '987'
+    $env:GITHUB_TOKEN = if ($Mode -eq 'Membership') { 'membership-token' } else { 'github-token' }
+    $env:PIPELINE_ID = '123'; $env:AZDO_TOKEN = 'test-token'
+    $env:REQUEST_BODY = ConvertTo-Json -InputObject $request -Depth 10 -Compress
+    $env:RUN_URL = ''; $env:HEAD_SHA = 'a' * 40; $env:BASE_SHA = 'b' * 40
+    $expectedToken = if ($Mode -eq 'Queue') { $env:AZDO_TOKEN } else { $env:GITHUB_TOKEN }
+    . $Change
+    ConvertTo-Json -InputObject $event -Depth 10 | Set-Content -LiteralPath $env:GITHUB_EVENT_PATH -Encoding utf8
+    Remove-Item -LiteralPath $env:GITHUB_OUTPUT -ErrorAction SilentlyContinue
+    $failure = $null
+    try { $stdout = @(& "$PSScriptRoot\pr-validation.ps1" -Mode $Mode) }
+    catch { $failure = "$_" }
+    if ($ExpectedError) {
+        Assert ($failure -and $failure -match $ExpectedError) "$Name failed for wrong reason: $failure"
+        Assert (!(Test-Path -LiteralPath $env:GITHUB_OUTPUT)) "$Name wrote outputs on failure"
+        Assert ($failure -notmatch 'test-token|github-token|membership-token') 'Leaked token'
+    }
+    else {
+        Assert (!$failure) "$Name failed: $failure"
+        Assert ($stdout.Count -eq 0) "$Name leaked stdout"
+        switch ($Mode) {
+            'Prepare' {
+                $lines = @(Get-Content -LiteralPath $env:GITHUB_OUTPUT)
+                Assert ($lines.Count -eq 3 -and $lines[1] -ceq "head=$('a' * 40)" -and $lines[2] -ceq "base=$('b' * 40)") 'Wrong snapshot outputs'
+                Assert-Snapshot (ConvertFrom-Json $lines[0].Substring(5))
+                Assert ($calls[0].Uri -ceq $permissionUri -and $calls[1].Uri -ceq $pullUri) 'Wrong requester or PR'
+            }
+            'Queue' {
+                Assert-Snapshot (ConvertFrom-Json $calls[0].Body)
+                Assert ((Get-Content -LiteralPath $env:GITHUB_OUTPUT -Raw).Trim() -ceq 'url=https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=456') 'Wrong run link'
+            }
+            'Report' {
+                $body = (ConvertFrom-Json $calls[0].Body).body
+                $expected = if ($env:RUN_URL) { '[F# Apex run](https://example.invalid/run) requested by @maintainer.' }
+                else { 'F# Apex request failed. [Workflow details](https://github.com/dotnet/fsharp/actions/runs/987).' }
+                Assert ($body.StartsWith($expected)) 'Wrong PR feedback'
+                Assert ($body.Contains('`/dart`') -and $body.Contains('`/pr-val`')) 'Missing retry guidance'
+                if ($env:RUN_URL) { Assert ($body.Contains("PR head: ``$env:HEAD_SHA``") -and $body.Contains("Base: ``$env:BASE_SHA``")) 'Missing snapshot identifiers' }
+            }
+        }
+        if ($CallCount -lt 0) { $CallCount = if ($Mode -eq 'Prepare') { 2 } else { 1 } }
+    }
+    if ($CallCount -ge 0) { Assert ($calls.Count -eq $CallCount) "$Name made unexpected HTTP calls" }
+    $script:count++
+    Write-Host "PASS: $Name"
+}
+
+try {
+    foreach ($command in '/dart', '/pr-val', " `n/pr-val `n") {
+        foreach ($authorRepo in 'dotnet/fsharp', 'contributor/fsharp') {
+            Test-Case "$command from $authorRepo" { $event.comment.body = $command; $pr.head.repo.full_name = $authorRepo }
+        }
+    }
+    foreach ($command in '/dart abc1234', 'please /dart', '`/dart`', "/dart`n/pr-val", '/dart-extra', '/DART', "/dart`nextra", '', $null, 42) {
+        Test-Case "reject command '$command'" { $event.comment.body = $command } 'standalone' -CallCount 0
+    }
+    foreach ($access in 'write', 'maintain', 'admin') { Test-Case "permission $access" { $permission.permission = $access } }
+    foreach ($access in 'read', 'triage', 'WRITE', '', $null, @('write')) { Test-Case "reject permission '$access'" { $permission.permission = $access } 'write access' -CallCount 1 }
+    Test-Case 'bot' { $event.comment.user.type = 'Bot' } 'user comments' -CallCount 0
+    Test-Case 'array user type' { $event.comment.user.type = @('User') } 'user comments' -CallCount 0
+    Test-Case 'issue' { $event.issue.Remove('pull_request') } 'dotnet/fsharp PRs' -CallCount 0
+    Test-Case 'false PR marker' { $event.issue.pull_request = $false } 'dotnet/fsharp PRs' -CallCount 0
+    Test-Case 'array event' { $event = @($event) } 'dotnet/fsharp PRs' -CallCount 0
+    Test-Case 'array permission response' { $responses[$permissionUri].Body = @($permission) } 'write access'
+    Test-Case 'array PR response' { $responses[$pullUri].Body = @($pr) } 'open PRs'
+    foreach ($repo in 'other/fsharp', 'dotnet/other', 'DotNet/fsharp') {
+        Test-Case "wrong repo $repo" { $env:GITHUB_REPOSITORY = $repo } 'dotnet/fsharp PRs' -CallCount 0
+    }
+    foreach ($id in '1; command', '42', $null, 0, -1, 1.5, $true, 9007199254740992) {
+        Test-Case "bad PR number '$id'" { $event.issue.number = $id } 'PR number' -CallCount 0
+    }
+    Test-Case 'closed PR' { $pr.state = 'closed' } 'open PRs'
+    Test-Case 'uppercase state' { $pr.state = 'OPEN' } 'open PRs'
+    Test-Case 'array state' { $pr.state = @('open') } 'open PRs'
+    Test-Case 'array target' { $pr.base.repo.full_name = @('dotnet/fsharp') } 'main'
+    Test-Case 'array branch' { $pr.base.ref = @('main') } 'main'
+    foreach ($branch in 'release/test', 'Main') { Test-Case "non-main $branch" { $pr.base.ref = $branch } 'main' }
+    Test-Case 'foreign target' { $pr.base.repo.full_name = 'other/fsharp' } 'main'
+    foreach ($side in 'head', 'base') {
+        foreach ($sha in 'abcdef0', $null, ('A' * 40), (('a' * 40) + "`n"), 123, @('a' * 40)) {
+            Test-Case "bad $side SHA '$sha'" { $pr[$side].sha = $sha } 'full head/base'
+        }
+    }
+    Test-Case 'membership original commenter' -Mode Membership
+    foreach ($status in 200, 201, 301, 302, 307, 308, 401, 403, 404, 429, 500) {
+        Test-Case "membership HTTP $status" { $responses[$memberUri].Status = $status } "HTTP $status" Membership
+    }
+    foreach ($endpoint in 'permissionUri', 'pullUri', 'memberUri', 'queueUri', 'reportUri') {
+        $mode = switch ($endpoint) { memberUri { 'Membership' }; queueUri { 'Queue' }; reportUri { 'Report' }; default { 'Prepare' } }
+        Test-Case "$endpoint network failure" { $responses[(Get-Variable $endpoint -ValueOnly)].Failure = $true } 'HTTP request failed' $mode
+        Test-Case "$endpoint API failure" { $responses[(Get-Variable $endpoint -ValueOnly)].Status = 500 } 'HTTP 500' $mode
+    }
+    Test-Case 'queue snapshot' -Mode Queue
+    Test-Case 'queue created' { $responses[$queueUri].Status = 201 } -Mode Queue
+    foreach ($json in '{"id":456.0}', '{"id":4.56e2}') {
+        Test-Case "integral JSON number $json" { $responses[$queueUri].Raw = $json } -Mode Queue
+    }
+    foreach ($pipeline in '', '0', '123/path', "123`n", ' 123', '+123') {
+        Test-Case "invalid pipeline '$pipeline'" { $env:PIPELINE_ID = $pipeline } 'Configure FSHARP_APEX_PIPELINE_ID' Queue 0
+    }
+    foreach ($token in '', ' ') { Test-Case 'missing queue token' { $env:AZDO_TOKEN = $token } 'access token' Queue 0 }
+    Test-Case 'invalid request JSON' { $env:REQUEST_BODY = '{' } 'JSON' Queue 0
+    foreach ($status in 301, 302, 303, 307, 308, 401, 403, 429, 500) {
+        Test-Case "queue HTTP $status" { $responses[$queueUri].Status = $status } "HTTP $status" Queue
+    }
+    foreach ($id in $null, 0, -1, 1.5, '456', $true, 9007199254740992, @(456)) {
+        Test-Case "invalid run ID '$id'" { $responses[$queueUri].Body.id = $id } 'valid run ID' Queue
+    }
+    foreach ($json in '{}', 'null', '[{"id":456}]', '{"id":[456]}', '{"id":{}}', '{"ID":456}') {
+        Test-Case "invalid run $json" { $responses[$queueUri].Raw = $json } 'valid run ID' Queue
+    }
+    Test-Case 'invalid response JSON' { $responses[$queueUri].Raw = '<html>' } 'JSON' Queue
+    Test-Case 'report success' { $env:RUN_URL = 'https://example.invalid/run' } -Mode Report
+    Test-Case 'report failure' -Mode Report
+    Write-Host "$count tests passed."
+}
+finally { Remove-Item -LiteralPath $root -Recurse -Force }

.github/scripts/pr-validation.ps1

@@ -0,0 +1,85 @@
+param([Parameter(Mandatory)][ValidateSet('Prepare', 'Membership', 'Queue', 'Report')][string]$Mode)
+
+$ErrorActionPreference = 'Stop'
+
+function Test-SafeId($Value) {
+    ($Value -is [long] -or $Value -is [int] -or $Value -is [double]) -and
+        $Value -gt 0 -and $Value -le 9007199254740991 -and [Math]::Truncate($Value) -eq $Value
+}
+
+function Invoke-Api($Uri, $Token, $Method = 'GET', $Body = $null, $ExpectedStatus = 200) {
+    $options = @{
+        Uri = $Uri; Method = $Method; Headers = @{ Authorization = "Bearer $Token"; Accept = 'application/json' }
+        UserAgent = 'fsharp-pr-validation'; ContentType = 'application/json'
+        MaximumRedirection = 0; SkipHttpErrorCheck = $true; ErrorAction = 'Stop'
+    }
+    if ($null -ne $Body) { $options.Body = ConvertTo-Json -InputObject $Body -Depth 10 -Compress }
+    try { $response = Invoke-WebRequest @options }
+    catch { throw "HTTP request failed for $Uri." }
+    if ($response.StatusCode -notin $ExpectedStatus) { throw "HTTP $($response.StatusCode) from $Uri." }
+    if ($response.StatusCode -ne 204) { ConvertFrom-Json -InputObject $response.Content -NoEnumerate }
+}
+
+if ($Mode -ne 'Queue') {
+    $event = Get-Content -LiteralPath $env:GITHUB_EVENT_PATH -Raw | ConvertFrom-Json -NoEnumerate
+    $prNumber = $event.issue.number
+    if ($env:GITHUB_REPOSITORY -cne 'dotnet/fsharp' -or $event -is [array] -or
+        $event.issue.pull_request -isnot [System.Management.Automation.PSCustomObject] -or !(Test-SafeId $prNumber)) {
+        throw 'Only dotnet/fsharp PRs with a valid PR number are supported.'
+    }
+    # Always use the original commenter, not the actor rerunning the workflow.
+    $login = [Uri]::EscapeDataString($event.comment.user.login)
+    $repoApi = 'https://api.github.com/repos/dotnet/fsharp'
+}
+
+switch ($Mode) {
+    'Prepare' {
+        if ($event.comment.user.type -isnot [string] -or $event.comment.user.type -cne 'User') { throw 'Only user comments can request validation.' }
+        if ($event.comment.body -isnot [string] -or $event.comment.body.Trim() -cnotmatch '\A/(dart|pr-val)\z') {
+            throw 'Use a standalone /dart or /pr-val, without a SHA argument.'
+        }
+        $permission = Invoke-Api "$repoApi/collaborators/$login/permission" $env:GITHUB_TOKEN
+        if ($permission -is [array] -or $permission.permission -isnot [string] -or
+            $permission.permission -cnotin 'write', 'maintain', 'admin') { throw 'The requester needs repository write access.' }
+        $pr = Invoke-Api "$repoApi/pulls/$prNumber" $env:GITHUB_TOKEN
+        if ($pr -is [array] -or $pr.state -isnot [string] -or $pr.state -cne 'open' -or
+            $pr.base.repo.full_name -isnot [string] -or $pr.base.repo.full_name -cne 'dotnet/fsharp' -or
+            $pr.base.ref -isnot [string] -or $pr.base.ref -cne 'main') {
+            throw 'Only open PRs targeting dotnet/fsharp main are supported.'
+        }
+        $headSha, $baseSha = $pr.head.sha, $pr.base.sha
+        foreach ($sha in $headSha, $baseSha) {
+            if ($sha -isnot [string] -or $sha -cnotmatch '\A[0-9a-f]{40}\z') { throw 'GitHub did not return full head/base commit SHAs.' }
+        }
+        $request = @{
+            resources = @{ repositories = @{ self = @{ refName = 'refs/heads/main'; version = $baseSha } } }
+            templateParameters = @{ prNumber = "$prNumber"; headSha = $headSha; baseSha = $baseSha }
+        }
+        $json = ConvertTo-Json -InputObject $request -Depth 10 -Compress
+        Add-Content -LiteralPath $env:GITHUB_OUTPUT -Encoding utf8 -Value "body=$json", "head=$headSha", "base=$baseSha"
+    }
+    'Membership' {
+        $null = Invoke-Api "https://api.github.com/orgs/microsoft/members/$login" $env:GITHUB_TOKEN -ExpectedStatus 204
+    }
+    'Queue' {
+        if ($env:PIPELINE_ID -cnotmatch '\A[1-9][0-9]*\z') { throw 'Configure FSHARP_APEX_PIPELINE_ID in the fsharp_pr_validation environment.' }
+        if ([string]::IsNullOrWhiteSpace($env:AZDO_TOKEN)) { throw 'Azure did not return an access token.' }
+        $request = ConvertFrom-Json -InputObject $env:REQUEST_BODY -NoEnumerate
+        $run = Invoke-Api "https://dev.azure.com/devdiv/DevDiv/_apis/pipelines/$env:PIPELINE_ID/runs?api-version=7.1" `
+            $env:AZDO_TOKEN -Method POST -Body $request -ExpectedStatus (200..299)
+        if ($run -is [array] -or $run.PSObject.Properties.Name -cnotcontains 'id' -or !(Test-SafeId $run.id)) {
+            throw 'Azure DevOps did not return a valid run ID.'
+        }
+        Add-Content -LiteralPath $env:GITHUB_OUTPUT -Encoding utf8 -Value "url=https://dev.azure.com/devdiv/DevDiv/_build/results?buildId=$($run.id)"
+    }
+    'Report' {
+        $workflowUrl = "$env:GITHUB_SERVER_URL/dotnet/fsharp/actions/runs/$env:GITHUB_RUN_ID"
+        $body = if ($env:RUN_URL) {
+            "[F# Apex run]($env:RUN_URL) requested by @$($event.comment.user.login).`n`nPR head: ``$env:HEAD_SHA```nBase: ``$env:BASE_SHA```n`nThe run will fail before building if the PR merge does not match this snapshot. Changes require a new ``/dart`` or ``/pr-val`` comment."
+        }
+        else {
+            "F# Apex request failed. [Workflow details]($workflowUrl). Use a standalone ``/dart`` or ``/pr-val`` on an open PR targeting ``main``; the requester needs repository write access and Microsoft-org membership."
+        }
+        $null = Invoke-Api "$repoApi/issues/$prNumber/comments" $env:GITHUB_TOKEN -Method POST -Body @{ body = $body } -ExpectedStatus 201
+    }
+}

.github/workflows/pr-validation.yml

@@ -0,0 +1,76 @@
+name: PR Validation (F# VS Integration)
+
+on:
+  issue_comment:
+    types: [created]
+
+permissions:
+  contents: read
+  pull-requests: write
+  id-token: write
+
+jobs:
+  validate-and-trigger:
+    if: |
+      github.repository == 'dotnet/fsharp' &&
+      github.event.issue.pull_request &&
+      (contains(github.event.comment.body, '/pr-val') || contains(github.event.comment.body, '/dart'))
+    runs-on: ubuntu-latest
+    environment: fsharp_pr_validation
+    defaults:
+      run:
+        shell: pwsh
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
+        with:
+          # issue_comment's SHA is on the default branch, never the PR branch.
+          ref: ${{ github.sha }}
+          persist-credentials: false
+
+      - name: Validate request and capture PR revision
+        id: request
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: .\.github\scripts\pr-validation.ps1 -Mode Prepare
+
+      - name: Get Microsoft membership lookup token
+        id: membership-token
+        uses: actions/create-github-app-token@fee1f7d63c2ff003460e3d139729b119787bc349 # v2
+        with:
+          app-id: ${{ secrets.MICROSOFT_MEMBERS_APP_ID }}
+          private-key: ${{ secrets.MICROSOFT_MEMBERS_APP_PRIVATE_KEY }}
+          owner: microsoft
+          permission-members: read
+
+      - name: Verify Microsoft organization membership
+        env:
+          GITHUB_TOKEN: ${{ steps.membership-token.outputs.token }}
+        run: .\.github\scripts\pr-validation.ps1 -Mode Membership
+
+      - name: Azure Login with OpenID Connect
+        uses: azure/login@8216e11d8cd9b42fe925c852af8e76311ff067ac # v2
+        with:
+          client-id: ${{ secrets.AZURE_CLIENT_ID }}
+          tenant-id: ${{ secrets.AZURE_TENANT_ID }}
+          allow-no-subscriptions: true
+
+      - name: Queue DartLab pipeline
+        id: queue
+        env:
+          PIPELINE_ID: ${{ vars.FSHARP_APEX_PIPELINE_ID }}
+          REQUEST_BODY: ${{ steps.request.outputs.body }}
+        run: |
+          $ErrorActionPreference = 'Stop'
+          $env:AZDO_TOKEN = az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query accessToken -o tsv
+          if ($LASTEXITCODE -ne 0 -or [string]::IsNullOrWhiteSpace($env:AZDO_TOKEN)) { throw 'Azure did not return an access token.' }
+          Write-Output "::add-mask::$env:AZDO_TOKEN"
+          .\.github\scripts\pr-validation.ps1 -Mode Queue
+
+      - name: Report request result
+        if: success() || failure()
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          RUN_URL: ${{ steps.queue.outputs.url }}
+          HEAD_SHA: ${{ steps.request.outputs.head }}
+          BASE_SHA: ${{ steps.request.outputs.base }}
+        run: .\.github\scripts\pr-validation.ps1 -Mode Report