diff --git a/.github/workflows/e2e-selfhosted.yml b/.github/workflows/e2e-selfhosted.yml index 9cf95bf19..a22e17c0e 100644 --- a/.github/workflows/e2e-selfhosted.yml +++ b/.github/workflows/e2e-selfhosted.yml @@ -34,7 +34,7 @@ on: type: choice default: all # Each value also selectable on its own; `all` covers every lane below. - options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, rad3] + options: [all, app-dev-gpu, strix-ubuntu, strix-windows, strix-wsl, rad3, mi350p] name_filter: description: "Scenario-name regex (cucumber --name); empty = full suite" type: string @@ -958,6 +958,132 @@ jobs: name: e2e-gpu-rad3-report path: tests/e2e-cucumber/results/ + e2e-gpu-mi350p: + name: E2E tests (MI350P) + # 90min: matches the other Instinct lane — same collapsed suite. + timeout-minutes: 90 + # `mi350p` names the hardware rather than reusing the generic `amd-gpu`: + # GitHub matches a job to any runner whose labels are a superset, so a shared + # label would let this card pick up MI300X-targeted work and vice versa. + runs-on: [self-hosted, linux, mi350p] + needs: [changes] + if: >- + always() + && needs.changes.result == 'success' + && ( + (github.event_name != 'workflow_dispatch' + && needs.changes.outputs.serve == 'true') + || (github.event_name == 'workflow_dispatch' + && (inputs.platform == 'all' || inputs.platform == 'mi350p')) + ) + continue-on-error: true + env: + E2E_SERVE_TIMEOUT_SECS: "300" + E2E_INCLUDE_NIGHTLY: "${{ inputs.include_nightly && '1' || '' }}" + # Full serve matrix at the merge-queue gate, like every other lane (see e2e-gpu). + E2E_MERGE_QUEUE: "${{ github.event_name == 'merge_group' && '1' || '' }}" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + # The pod is usually fresh (the runner scales to zero between jobs), but the + # work volume is not, and two jobs can share one pod's life. Same reclaim as + # e2e-gpu, scoped to e2e leftovers only. + - name: Reclaim GPU from stray E2E processes + run: | + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # Bounded wait, as in e2e-gpu. The pod is admitted for exactly one card, so + # rocm-smi sees that card alone. The floor matches the other Instinct lane + # rather than scaling with the 144GB capacity: it only has to be high enough + # to catch a leftover serve still holding the card, and low enough that a + # clean pod passes on the first poll. + - name: GPU preflight (bounded wait for an available GPU) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + exit 1 + + # cache: false for the same reason as e2e-gpu — the build cache is kept in + # CARGO_TARGET_DIR on the work volume, not in GitHub's cache service. + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + + - name: Run E2E tests on MI350P + run: | + # $RUNNER_WORKSPACE is on a volume that survives the runner being scaled + # away, so the cargo target dir and the shared caches are still warm on + # the next job even though the container is not. + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # A SEPARATE volume is mounted at exactly this path by the runner's + # deployment. Keeping the ~23GB uv cache off the work volume is + # deliberate; if you change this path, change that deployment too. + export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + # Pre-warm one shared runtime in place before the suite. Not an + # optimization — `install sdk` bakes absolute paths into the runtime + # manifest, so installing anywhere temporary breaks every later serve. + # See e2e-gpu for the full rationale. + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # See the e2e-gpu lane for why the e2e-test-hooks feature must match + # what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" + + # Pre-warm once, serially, in place (no mv/symlink), and refresh it when + # the channel index has moved on — the tree is a cache, not a one-shot. + # See the e2e-gpu lane and `xtask e2e-prewarm`. + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ + cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" + + # Optional scenario-name filter for a scoped manual dispatch — a cheap + # way to exercise this lane against a single scenario without the full suite. + NAME_FILTER="${{ github.event.inputs.name_filter }}" + if [ -n "$NAME_FILTER" ]; then + echo "name filter active: $NAME_FILTER" + cargo xtask e2e -- --name "$NAME_FILTER" + else + cargo xtask e2e + fi + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-mi350p-report + path: tests/e2e-cucumber/results/ + # Consolidate this workflow's self-hosted platform reports into one GPU-side # cross-platform grid (Summary + merged HTML). Distinct name from ci.yml's # required `E2E consolidated report` so it does NOT collide with that required @@ -975,6 +1101,7 @@ jobs: - e2e-gpu-strix-windows - e2e-wsl - e2e-gpu-rad3 + - e2e-gpu-mi350p # Gate on `serve`: every lane this report consolidates (the GPU jobs) is now # serve-gated, so a serve-only change runs them and their report must still be # produced. On dispatch `serve` is unset, so also run when the trigger was diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 373e8ff41..2a18416cf 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -549,6 +549,95 @@ jobs: name: e2e-gpu-rad3-report path: tests/e2e-cucumber/results/ + e2e-gpu-nightly-mi350p: + name: E2E tests (MI350P, incl. nightly-only) + timeout-minutes: 90 + runs-on: [self-hosted, linux, mi350p] + continue-on-error: true + env: + E2E_INCLUDE_NIGHTLY: "1" + E2E_SERVE_TIMEOUT_SECS: "300" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + + - name: Reclaim GPU from stray E2E processes + run: | + pkill -f '/tmp/rocm-e2e.*llama-server' 2>/dev/null || true + pkill -f '/tmp/rocm-e2e.*vllm serve' 2>/dev/null || true + pkill -f 'e2e-shared.*llama-server' 2>/dev/null || true + pkill -f '__engine-serve-http.*rocm-e2e' 2>/dev/null || true + pkill -f 'e2e-target/release/rocm daemon' 2>/dev/null || true + rm -rf /tmp/rocm-e2e-* 2>/dev/null || true + echo "reclaimed" + + # Bounded wait for a free GPU. Floor matches the per-PR MI350P lane and the + # other Instinct lane — high enough to catch a leftover serve, low enough + # that a clean pod passes on the first poll. + - name: GPU preflight (bounded wait for an available GPU) + run: | + MIN_FREE_GIB="${GPU_PREFLIGHT_MIN_FREE_GIB:-16}" + CEILING_SECS="${GPU_PREFLIGHT_CEILING_SECS:-90}" + min_free=$(( MIN_FREE_GIB * 1024 * 1024 * 1024 )) + deadline=$(( SECONDS + CEILING_SECS )) + reason="rocm-smi never returned within its timeout (driver wedged or GPU absent)" + while [ "$SECONDS" -lt "$deadline" ]; do + out=$(timeout 15 rocm-smi --showmeminfo vram 2>/dev/null) || { sleep 5; continue; } + total=$(printf '%s\n' "$out" | grep -i 'VRAM Total Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + used=$(printf '%s\n' "$out" | grep -i 'VRAM Total Used Memory' | sed 's/.*: *//' | grep -oE '[0-9]+' | tail -1) + if [ -z "$total" ] || [ -z "$used" ]; then + reason="rocm-smi returned no VRAM figures (no AMD GPU detected)" + sleep 5; continue + fi + free=$(( total - used )) + if [ "$free" -ge "$min_free" ]; then + echo "GPU ready: $(( free / 1024 / 1024 / 1024 )) GiB free (>= ${MIN_FREE_GIB} GiB)." + exit 0 + fi + reason="VRAM never dropped below the floor: only $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB) — a serve is likely still holding the GPU" + echo "waiting: $(( free / 1024 / 1024 / 1024 )) GiB free (< ${MIN_FREE_GIB} GiB)…" + sleep 5 + done + echo "::error::GPU preflight failed after ${CEILING_SECS}s: ${reason}" + exit 1 + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0 + with: + cache: false + + - name: Run full E2E on MI350P (incl. nightly-only) + run: | + # $RUNNER_WORKSPACE is on a volume that survives the runner being scaled + # away, so the cargo target dir and shared caches stay warm. + export CARGO_TARGET_DIR="$RUNNER_WORKSPACE/e2e-target" + export E2E_SHARED_CACHE_DIR="$RUNNER_WORKSPACE/e2e-shared" + # Separate volume mounted at exactly this path by the runner's + # deployment; if you change this path, change that deployment too. + export E2E_SHARED_UV_CACHE_DIR="/var/tmp/rocm-e2e-uv-cache" + prewarm="$RUNNER_WORKSPACE/e2e-prewarm-multi-arch-v2" + export E2E_SHARED_RUNTIMES_DIR="$prewarm/data/runtimes" + + # See the e2e-gpu lane in e2e-selfhosted.yml for why the + # e2e-test-hooks feature must match what `cargo xtask e2e` would build. + cargo build --release -p rocm -p rocmd --features rocm/e2e-test-hooks + export ROCM_CLI_BINARY="$CARGO_TARGET_DIR/release/rocm" + export ROCM_CLI_ROCMD_BINARY="$CARGO_TARGET_DIR/release/rocmd" + + # Pre-warm once, serially, in place (no mv/symlink), and refresh it when + # the channel index has published a newer runtime — the tree is a cache, + # not a one-shot. See `xtask e2e-prewarm`. + HF_HOME="$E2E_SHARED_CACHE_DIR/huggingface" \ + UV_CACHE_DIR="$E2E_SHARED_UV_CACHE_DIR" \ + cargo xtask e2e-prewarm --channel release --prewarm-dir "$prewarm" + + cargo xtask e2e + + - name: Upload E2E report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: e2e-gpu-mi350p-report + path: tests/e2e-cucumber/results/ + # Strix Halo counterpart to e2e-gpu-nightly. The same @nightly large-model # scenario serves the platform-specific Lemonade GGUF on this host. Keep the # runner's storage and runtime setup aligned with ci.yml's proven Strix Ubuntu job. @@ -910,6 +999,7 @@ jobs: needs: - e2e-gpu-nightly - e2e-gpu-nightly-rad3 + - e2e-gpu-nightly-mi350p - e2e-gpu-nightly-strix - e2e-gpu-nightly-strix-windows - e2e-wsl-nightly diff --git a/crates/e2e-report/src/lib.rs b/crates/e2e-report/src/lib.rs index db0c4ac95..5d0d3e157 100644 --- a/crates/e2e-report/src/lib.rs +++ b/crates/e2e-report/src/lib.rs @@ -413,6 +413,7 @@ fn parse_descriptor(name: &str) -> Descriptor { "" | "report" => ("Mock", "Linux"), "gpu" => ("MI300X", "Linux"), "gpu-rad3" => ("R9700", "Linux"), + "gpu-mi350p" => ("MI350P", "Linux"), "gpu-strix-ubuntu" => ("Strix Halo", "Ubuntu"), "gpu-strix-windows" => ("Strix Halo", "Windows"), // Same silicon again, third host boundary: an Ubuntu distro under WSL2 on @@ -1298,8 +1299,10 @@ pub fn consolidated_summary_markdown(inputs: &[(String, PathBuf)]) -> String { forwarding — with no GPU, no model download, and no engine process, so it \ runs on a GitHub-hosted runner. It **gates the PR**: it runs on every push, \ and if it fails the PR's required check goes red and the PR cannot merge. \ - **MI300X / Strix Halo** run on real self-hosted GPU hardware with real \ - engines. They are **non-blocking**: they still run and are reported here, but \ + **Every other platform** is a real self-hosted GPU host running real \ + engines — the rows below are the list, so naming them here would only go \ + stale as lanes are added. They are **non-blocking**: they still run and \ + are reported here, but \ a failure does NOT block the PR from merging (the hardware/runners are still \ being proven out, so their results are informational rather than a merge \ gate).\n\n\ @@ -1935,7 +1938,7 @@ fn legend() -> Markup { "and gates the PR." } li { - b { "MI300X / Strix Halo" } + b { "Every other platform" } " — real self-hosted GPU hardware; non-blocking while proven out." } li { @@ -2219,6 +2222,7 @@ mod tests { ("e2e-report", "Mock", "Linux"), ("e2e-gpu-report", "MI300X", "Linux"), ("e2e-gpu-rad3-report", "R9700", "Linux"), + ("e2e-gpu-mi350p-report", "MI350P", "Linux"), ("e2e-gpu-strix-ubuntu-report", "Strix Halo", "Ubuntu"), ("e2e-gpu-strix-windows-report", "Strix Halo", "Windows"), // Must not fall through to `fallback_descriptor`, which would render diff --git a/docs/ci-hardware-testing.md b/docs/ci-hardware-testing.md index 7942a434f..4a0a5fca5 100644 --- a/docs/ci-hardware-testing.md +++ b/docs/ci-hardware-testing.md @@ -37,6 +37,7 @@ separate tier flag or tag filter to maintain. | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on native Windows 11 | self-hosted `[self-hosted, windows, strix-halo, native]` | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo (gfx1151) on Ubuntu under WSL2 | self-hosted `[self-hosted, linux, strix-halo, wsl]` | | `e2e-gpu-rad3` | `e2e-selfhosted.yml` | Radeon AI PRO R9700 (gfx1201) on Linux | self-hosted `[self-hosted, linux, r9700]` | +| `e2e-gpu-mi350p` | `e2e-selfhosted.yml` | MI350P (AMD Instinct, gfx950) on Linux | self-hosted `[self-hosted, linux, mi350p]` | The Strix Halo lanes pin the extra `native` label because two Linux runners share the `strix-halo` label (a native host and a WSL host) and the jobs' @@ -55,10 +56,10 @@ named for. resolve to skip here, and known bugs resolve to xfail from `expectations.toml`. It is a required check and must stay green. -The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-rad3`) run on AMD GPU systems, so they exercise -host/GPU detection, engine `detect`/`capabilities`, and live serving scenarios -that the mock job cannot. GPU availability is advisory in the WSL lane, as +The self-hosted jobs (`e2e-gpu`, `e2e-gpu-strix-ubuntu`, `e2e-gpu-strix-windows`, +`e2e-wsl`, `e2e-gpu-rad3`, and `e2e-gpu-mi350p`) run on AMD GPU systems, so they +exercise host/GPU detection, engine `detect`/`capabilities`, and live serving +scenarios that the mock job cannot. GPU availability is advisory in the WSL lane, as described below. `e2e-wsl` runs on an Ubuntu distro hosted in WSL2 on the Strix Halo Windows box @@ -100,10 +101,11 @@ reports — including partial or failed runs — by scenario id into one HTML re and GitHub step summary. The lane artifacts are named canonically (`e2e-report`, `e2e-gpu-report`, -`e2e-gpu-rad3-report`, `e2e-gpu-strix-ubuntu-report`, `e2e-gpu-strix-windows-report`, -`e2e-gpu-strix-wsl-report`) in every workflow, because the report derives each -platform's name and OS from the artifact name. An unrecognised name renders as a -guessed platform on Linux, which would report a Windows lane as Linux; `xtask`'s +`e2e-gpu-rad3-report`, `e2e-gpu-mi350p-report`, `e2e-gpu-strix-ubuntu-report`, +`e2e-gpu-strix-windows-report`, `e2e-gpu-strix-wsl-report`) in every workflow, +because the report derives each platform's name and OS from the artifact name. +An unrecognised name renders as a guessed platform on Linux, which would report +a Windows lane as Linux; `xtask`'s `every_uploaded_e2e_artifact_has_a_name_the_report_can_label` guards against it. ## Triggers @@ -128,11 +130,11 @@ They can also be triggered manually via `e2e-selfhosted.yml`'s `workflow_dispatch`, independent of the `serve` gate, with these inputs: - `platform` (choice: `all`, `app-dev-gpu`, `strix-ubuntu`, `strix-windows`, - `strix-wsl`, `rad3`) — which self-hosted job(s) to run. `app-dev-gpu` maps to - `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to - `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, and `rad3` to - `e2e-gpu-rad3`. (The mock lane has its own `platform` input on `ci.yml`; it is - not part of this workflow.) + `strix-wsl`, `rad3`, `mi350p`) — which self-hosted job(s) to run. `app-dev-gpu` + maps to `e2e-gpu`, `strix-ubuntu` to `e2e-gpu-strix-ubuntu`, `strix-windows` to + `e2e-gpu-strix-windows`, `strix-wsl` to `e2e-wsl`, `rad3` to + `e2e-gpu-rad3`, and `mi350p` to `e2e-gpu-mi350p`. (The mock lane has its own + `platform` input on `ci.yml`; it is not part of this workflow.) - `name_filter` (string) — a scenario-name regex forwarded to the cucumber harness (`cargo xtask e2e -- --name `) so a dispatch can run a single scenario instead of the full suite. Empty runs everything applicable @@ -214,8 +216,9 @@ the pre-warm block is duplicated across multiple jobs in two shells; ## Blocking vs. non-blocking The self-hosted jobs — `e2e-gpu`, `e2e-gpu-strix-ubuntu`, -`e2e-gpu-strix-windows`, `e2e-wsl`, and `e2e-gpu-rad3` — all run with `continue-on-error: true`, so a -hardware failure that RUNS never gates a PR merge. Their results still surface +`e2e-gpu-strix-windows`, `e2e-wsl`, `e2e-gpu-rad3`, and `e2e-gpu-mi350p` — all run with +`continue-on-error: true`, so a hardware failure that RUNS never gates a PR +merge. Their results still surface in the self-hosted consolidated report for visibility. ### Timeouts on the shared Strix box diff --git a/tests/e2e-cucumber/README.md b/tests/e2e-cucumber/README.md index 12a21bca6..5b9b6b7b1 100644 --- a/tests/e2e-cucumber/README.md +++ b/tests/e2e-cucumber/README.md @@ -172,6 +172,8 @@ self-hosted runner can never stall `ci.yml`'s merge-required checks: | `e2e-gpu-strix-ubuntu` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu (self-hosted) | no | | `e2e-gpu-strix-windows` | `e2e-selfhosted.yml` | Strix Halo / Windows (self-hosted) | no | | `e2e-wsl` | `e2e-selfhosted.yml` | Strix Halo / Ubuntu under WSL2 (self-hosted) | no | +| `e2e-gpu-rad3` | `e2e-selfhosted.yml` | Radeon R9700 (self-hosted) | no | +| `e2e-gpu-mi350p` | `e2e-selfhosted.yml` | MI350P (self-hosted) | no | The blocking mock job passes when every applicable scenario is pass-or-xfail with no XPASS or unexpected failure; the GPU jobs are non-blocking. Each workflow @@ -182,12 +184,14 @@ Ubuntu distro under WSL2 on the Strix Halo box, so it is the only lane that exercises `@requires-wsl` scenarios; `@requires-bare-metal` scenarios resolve to skip there. -The nightly workflow runs non-blocking jobs — MI300X, Radeon R9700, and Strix -Halo on Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`, then -consolidates them into the same cross-platform grid. The -shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on MI300X and -the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` checkpoint -through Lemonade on Strix Halo. +The nightly workflow covers the same hardware as the table above, as +non-blocking lanes (`e2e-gpu-nightly`, `e2e-gpu-nightly-rad3`, +`e2e-gpu-nightly-mi350p`, `e2e-gpu-nightly-strix`, +`e2e-gpu-nightly-strix-windows`, `e2e-wsl-nightly`) with +`E2E_INCLUDE_NIGHTLY=1`, then consolidates them into the same cross-platform +grid. The shared large-model scenario serves `Qwen/Qwen3.6-27B` through vLLM on +MI300X and the hardware-verified `unsloth/Qwen3.6-35B-A3B-GGUF:UD-Q4_K_XL` +checkpoint through Lemonade on Strix Halo. Use the self-hosted E2E workflow dispatch to run either model independently on a ref (the GPU platform / `include_nightly` / `name_filter` inputs live on diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index 90bc4f2bb..3b01a5d64 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -532,7 +532,13 @@ fn derive_platform_slug( fn platform_hardware_slug(gfx_target: &str) -> String { let family = normalize_family(gfx_target); - if family.ends_with("-dcgpu") { + // Two distinct data-center parts normalize to a `-dcgpu` family, so match the + // family rather than the suffix: a suffix test reports gfx950 hardware as + // `mi300x`, which would file its results in the MI300X column of the report + // grid instead of its own. + if family == "gfx950-dcgpu" { + "mi350p".to_owned() + } else if family.ends_with("-dcgpu") { "mi300x".to_owned() } else if family.starts_with("gfx115") { "strix-halo".to_owned() @@ -795,10 +801,28 @@ Local model engines #[test] fn platform_slug_derivation() { assert_eq!(derive_platform_slug(false, None, "other", false), "mock"); - assert_eq!( - derive_platform_slug(true, Some("gfx942"), "linux", false), - "mi300x" - ); + // gfx950 normalizes to a `-dcgpu` family like gfx94x does, but it is a + // different part with its own lane and report column — it must not be + // slugged as mi300x. + // + // Cover every form that reaches `platform_hardware_slug`, not just the + // bare target the probe happens to emit today: the family label takes + // `normalize_family`'s early `-dcgpu` return, while the suffixed form + // depends on its `starts_with` — an `==` "tidy-up" there would silently + // send a real gfx950 host back into the `mi300x` column. + for (gfx_target, expected) in [ + ("gfx950", "mi350p"), + ("gfx950-dcgpu", "mi350p"), + ("gfx950:sramecc+:xnack-", "mi350p"), + ("gfx942", "mi300x"), + ("gfx94X-dcgpu", "mi300x"), + ] { + assert_eq!( + derive_platform_slug(true, Some(gfx_target), "linux", false), + expected, + "gfx target `{gfx_target}` must slug as `{expected}`" + ); + } // Strix Halo: same gfx1151 silicon on both OSes → distinct slugs so the // report grid gets a column per platform, not a collision. assert_eq!( diff --git a/xtask/src/e2e_report.rs b/xtask/src/e2e_report.rs index 3a70f99cd..6fda5e74a 100644 --- a/xtask/src/e2e_report.rs +++ b/xtask/src/e2e_report.rs @@ -132,6 +132,7 @@ fn label_for_root_report(dir: &Path) -> String { Some("mock") => "e2e-report".to_owned(), Some("mi300x") => "e2e-gpu-report".to_owned(), Some("gfx1201") => "e2e-gpu-rad3-report".to_owned(), + Some("mi350p") => "e2e-gpu-mi350p-report".to_owned(), Some("strix-halo-linux") => "e2e-gpu-strix-ubuntu-report".to_owned(), Some("strix-halo-windows") => "e2e-gpu-strix-windows-report".to_owned(), // This workflow is statically pinned to the Strix WSL runner. A bare @@ -188,6 +189,7 @@ mod tests { "e2e-report", "e2e-gpu-report", "e2e-gpu-rad3-report", + "e2e-gpu-mi350p-report", "e2e-gpu-strix-ubuntu-report", "e2e-gpu-strix-windows-report", "e2e-gpu-strix-wsl-report", @@ -300,6 +302,7 @@ mod tests { for (slug, expected) in [ ("mi300x", "e2e-gpu-report"), ("gfx1201", "e2e-gpu-rad3-report"), + ("mi350p", "e2e-gpu-mi350p-report"), ("strix-halo-linux", "e2e-gpu-strix-ubuntu-report"), ("strix-halo-windows", "e2e-gpu-strix-windows-report"), ("strix-halo-wsl", "e2e-gpu-strix-wsl-report"), diff --git a/xtask/src/workflow_contract.rs b/xtask/src/workflow_contract.rs index 05b8bc87d..5199feace 100644 --- a/xtask/src/workflow_contract.rs +++ b/xtask/src/workflow_contract.rs @@ -1006,14 +1006,68 @@ trigger-a-workflow#triggering-a-workflow-from-a-workflow" ); } + // The README's own lane lists were the one hand-copied hole in this + // otherwise derived net: the nightly sentence was asserted by literal + // string match against a copy in this test source, so both could go + // stale together while the guard stayed green — which is exactly what + // happened when the R9700 lane landed. Derive both from the YAML. + // + // Whitespace is normalized first because these are prose sentences and + // a table, wrapped for readability; the lane lists must survive a + // reflow that does not change what the doc says. let readme = std::fs::read_to_string(repo_root().join("tests/e2e-cucumber/README.md")) .expect("read E2E README"); + let readme_flat = normalized_whitespace(&readme); + + let nightly = read_workflow("nightly.yml"); + let nightly_lanes = self_hosted_e2e_jobs(&nightly); assert!( - normalized_whitespace(&readme).contains( - "The nightly workflow runs non-blocking jobs — MI300X, Radeon R9700, and Strix Halo on Ubuntu, Windows, and WSL2 — with `E2E_INCLUDE_NIGHTLY=1`" - ), - "E2E README must identify every nightly job platform" + !nightly_lanes.is_empty(), + "expected at least one self-hosted job in nightly.yml (extractor sanity check)" + ); + let nightly_ids: Vec = nightly_lanes.iter().map(|(job, _)| job.clone()).collect(); + assert_eq!( + backticked_list_between(&readme_flat, "as non-blocking lanes (", ") with"), + nightly_ids, + "the E2E README must name every nightly self-hosted lane, in workflow order" + ); + + // The README's CI job table is the per-PR view: the blocking mock job + // from ci.yml, then one row per self-hosted lane. Only the self-hosted + // rows are derivable, so the mock row is matched by workflow and the + // rest are compared against e2e-selfhosted.yml. + let readme_rows = markdown_table_rows(&readme, "| Job | Workflow | Platform | Blocking |"); + let (mock_rows, self_hosted_rows): (Vec<_>, Vec<_>) = readme_rows + .into_iter() + .partition(|row| row.get(1).is_some_and(|workflow| workflow == "`ci.yml`")); + assert_eq!( + mock_rows + .iter() + .map(|row| row[0].clone()) + .collect::>(), + vec!["`e2e`".to_owned()], + "the README CI job table must carry exactly one blocking mock row" + ); + assert_eq!( + self_hosted_rows + .iter() + .map(|row| row[0].clone()) + .collect::>(), + lanes + .iter() + .map(|(job, _)| format!("`{job}`")) + .collect::>(), + "the README CI job table must have one row per self-hosted job in \ + e2e-selfhosted.yml, in workflow order" ); + for row in &self_hosted_rows { + assert_eq!( + row[3], "no", + "self-hosted lane `{}` is continue-on-error, so the README must not \ + document it as blocking", + row[0] + ); + } } #[test]