diff --git a/.github/workflows/benchmark-clawbench.yml b/.github/workflows/benchmark-clawbench.yml index 3d87ee99..9f035ab4 100644 --- a/.github/workflows/benchmark-clawbench.yml +++ b/.github/workflows/benchmark-clawbench.yml @@ -220,7 +220,7 @@ jobs: BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }} BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }} HARBOR_VERSION: "0.21.0" - HARBOR_HYPEMAN_VERSION: "0.1.1" + HARBOR_HYPEMAN_VERSION: "0.1.2" CODEX_BENCHMARK_MODEL: gpt-5.6-luna CODEX_BENCHMARK_VERSION: "0.120.0" CLAUDE_BENCHMARK_MODEL: claude-sonnet-5 @@ -342,7 +342,8 @@ jobs: run_arm() { local checkout=$1 arm=$2 job_name=$3 set +e - "$checkout/benchmarks/harbor/clawbench/run.sh" \ + KERNEL_MCP_BENCHMARK_SOURCE_ROOT="$checkout" \ + "$GITHUB_WORKSPACE/harness/benchmarks/harbor/clawbench/run.sh" \ "$BENCHMARK_AGENT" \ "$BENCHMARK_TASK" \ "$job_name" \ @@ -479,4 +480,4 @@ jobs: [[ "$BASELINE_STATUS" == "0" ]] [[ "$PUBLISH_OUTCOME" == "success" ]] [[ "$REPORT_OUTCOME" == "success" ]] - jq -e 'all(.arms[]; .scored > 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null + jq -e 'all(.arms[]; .complete == true)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 771e7b6e..7b7b2ce2 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -16,7 +16,7 @@ The image records the current Git SHA, and the generated task records the ClawBe ## Requirements -- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1 +- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.2 - Hypeman CLI credentials - a ClawBench checkout containing pinned commit `c7feaa2` - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth @@ -57,6 +57,8 @@ Codex defaults to version `0.120.0` with `gpt-5.6-luna`. Claude Code defaults to Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to six hours. Set `HARBOR_BENCHMARK_TIMEOUT` to override either limit. Set `HARBOR_JOBS_DIR` to choose where Harbor writes results. +The runner retries a whole isolated trial up to five times for transient Hypeman connection, timeout, and exec-stream failures. Set `HARBOR_MAX_RETRIES` to override that limit. Per-request SDK retries remain disabled because transparently retrying instance or image creation can duplicate a request whose first response was lost. + ## GitHub Actions The `Benchmark ClawBench` workflow runs the complete suite weekly and on demand. Select it from the Actions tab and provide either a same-repository PR number or a ref. Comparison runs benchmark the candidate SHA against its merge base so unrelated changes on the target branch do not affect the delta. Harbor, Hypeman, agent, and model versions are pinned by the workflow and each arm's observed agent configuration appears in the report. @@ -101,3 +103,5 @@ BRAINTRUST_PROJECT=kernel-mcp-server-benchmarks \ ``` The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Re-publication replaces the rows and refreshes experiment metadata. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments. + +Reports suppress comparison deltas when either arm has an infrastructure failure or ungraded trial. The workflow fails unless every intended trial is graded, while still retaining the incomplete report for diagnosis. diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 93a752a9..1feda7e2 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -15,10 +15,11 @@ agent=${1:-} [[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage task_id=${2:-v2-1134-chapter-finder-redcross} -repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) -benchmark_dir="$repo_root/benchmarks/harbor" -image_env="$benchmark_dir/.image.env" -clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench} +harness_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd) +source_root=${KERNEL_MCP_BENCHMARK_SOURCE_ROOT:-$harness_root} +benchmark_dir="$harness_root/benchmarks/harbor" +image_env="$source_root/benchmarks/harbor/.image.env" +clawbench_repo=${CLAWBENCH_REPO:-$harness_root/../ClawBench} clawbench_ref=${CLAWBENCH_REF:-c7feaa2435ca8115c0762c44e13885fe5adf3e98} [[ -f "$image_env" ]] || { @@ -69,7 +70,7 @@ case "$agent" in esac harbor_version=${HARBOR_VERSION:-0.21.0} -harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.1} +harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.2} export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com} @@ -161,6 +162,12 @@ timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_time --job-name "$job_name" \ --jobs-dir "$jobs_dir" \ --n-concurrent "${HARBOR_N_CONCURRENT:-1}" \ - --max-retries 0 \ + --max-retries "${HARBOR_MAX_RETRIES:-5}" \ + --retry-include APITimeoutError \ + --retry-include APIConnectionError \ + --retry-include RateLimitError \ + --retry-include InternalServerError \ + --retry-include ConnectionRefusedError \ + --retry-include ExecProtocolError \ --delete \ --yes diff --git a/benchmarks/harbor/publish-braintrust.ts b/benchmarks/harbor/publish-braintrust.ts index fc342d75..c6fa1c1b 100644 --- a/benchmarks/harbor/publish-braintrust.ts +++ b/benchmarks/harbor/publish-braintrust.ts @@ -354,13 +354,17 @@ export async function publishBenchmark( experimentName: string, apiKey: string, ): Promise> { - const ungraded = arms + const incomplete = arms .map(summarizeArm) - .filter((summary) => summary.scored === 0) - .map((summary) => summary.arm); - if (ungraded.length > 0) { + .filter((summary) => !summary.complete); + if (incomplete.length > 0) { throw new Error( - `Cannot publish benchmark without graded trials for: ${ungraded.join(", ")}`, + `Cannot publish incomplete benchmark arms: ${incomplete + .map( + (summary) => + `${summary.arm} (${summary.incompleteReasons.join(", ")})`, + ) + .join("; ")}`, ); } diff --git a/benchmarks/harbor/report.ts b/benchmarks/harbor/report.ts index cb91a6ce..f2d3c6cd 100644 --- a/benchmarks/harbor/report.ts +++ b/benchmarks/harbor/report.ts @@ -89,12 +89,14 @@ export function renderMarkdown( ): string { const lines = ["", `## ${title}`]; const failed = Object.entries(statuses).filter(([, status]) => status !== 0); - const ungraded = summaries.filter((summary) => summary.scored === 0); - const incomplete = failed.length > 0 || ungraded.length > 0; + const incomplete = + failed.length > 0 || summaries.some((summary) => !summary.complete); if (incomplete) { const reasons = [ ...failed.map(([arm, status]) => `${arm} exited ${status}`), - ...ungraded.map((summary) => `${summary.arm} produced no graded trials`), + ...summaries.flatMap((summary) => + summary.incompleteReasons.map((reason) => `${summary.arm} ${reason}`), + ), ]; lines.push( "", @@ -103,12 +105,12 @@ export function renderMarkdown( } lines.push( "", - "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", - "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|", + "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Retries | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |", + "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|", ); for (const summary of summaries) { lines.push( - `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, + `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.retries} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`, ); } diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index 7634ad6d..a0774091 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -114,6 +114,24 @@ function fixture(): string { return root; } +function completeArm() { + const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const failed = arm.trials[1]; + failed.error = undefined; + failed.errorClass = undefined; + failed.rewards = { reward: 0, intercepted: 0 }; + failed.scores = { + accuracy: 0, + false_positive_rate: 0, + false_negative_rate: 1, + infra_error_rate: 0, + intercepted: 0, + reward: 0, + ungraded_rate: 0, + }; + return arm; +} + describe("Harbor result ingestion", () => { test("keeps infrastructure errors out of task-quality scores", () => { const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); @@ -146,13 +164,35 @@ describe("Harbor result ingestion", () => { strict: 0, intercepted: 1, infraErrors: 1, + retries: 0, ungraded: 0, + complete: false, + incompleteReasons: ["scored 1/2 trials", "had 1 infrastructure failure"], kernelMcpValid: 1, medianCalls: 1, totalCostUsd: 0.01, }); }); + test("requires every intended trial to be graded", () => { + expect(summarizeArm(completeArm()).complete).toBe(true); + + const missing = completeArm(); + missing.nTotalTrials = 3; + expect(summarizeArm(missing)).toMatchObject({ + complete: false, + incompleteReasons: ["scored 2/3 trials"], + }); + + const empty = completeArm(); + empty.nTotalTrials = 0; + empty.trials = []; + expect(summarizeArm(empty)).toMatchObject({ + complete: false, + incompleteReasons: ["had no intended trials"], + }); + }); + test("builds deterministic root, llm, and tool spans", () => { const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); const first = buildExperimentEvents([arm], "test-experiment"); @@ -192,7 +232,7 @@ describe("Harbor result ingestion", () => { }); test("re-publishes the same rows and spans by deterministic ID", async () => { - const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); + const arm = completeArm(); const originalFetch = globalThis.fetch; const inserts: string[][] = []; const metadataUpdates: unknown[] = []; @@ -258,14 +298,12 @@ describe("Harbor result ingestion", () => { } }); - test("does not publish arms without graded trials", async () => { + test("does not publish incomplete arms", async () => { const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); for (const trial of arm.trials) trial.rewards = {}; await expect( publishBenchmark([arm], "project", "experiment", "api-key"), - ).rejects.toThrow( - "Cannot publish benchmark without graded trials for: candidate", - ); + ).rejects.toThrow("Cannot publish incomplete benchmark arms: candidate"); }); test("uses the lenient reward per trial and reports incomplete arms", () => { @@ -273,19 +311,7 @@ describe("Harbor result ingestion", () => { key: "reward_lenient", value: 1, }); - const arm = readBenchmarkArm({ name: "candidate", path: fixture() }); - const second = arm.trials[1]; - second.error = undefined; - second.errorClass = undefined; - second.rewards = { reward: 0 }; - second.scores = { - accuracy: 0, - false_positive_rate: 0, - false_negative_rate: 1, - infra_error_rate: 0, - reward: 0, - ungraded_rate: 0, - }; + const arm = completeArm(); const summary = summarizeArm(arm); expect(summary.scored).toBe(2); expect(summary.lenient).toBe(1); @@ -300,11 +326,38 @@ describe("Harbor result ingestion", () => { ]), ).toContain("+0.1 lenient"); const ungraded = renderMarkdown("test", [ - { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials }, - { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials }, + { + ...summary, + arm: "candidate", + scored: 0, + ungraded: summary.trials, + complete: false, + incompleteReasons: ["had 2 ungraded trials"], + }, + { + ...summary, + arm: "baseline", + scored: 0, + ungraded: summary.trials, + complete: false, + incompleteReasons: ["had 2 ungraded trials"], + }, ]); - expect(ungraded).toContain("candidate produced no graded trials"); + expect(ungraded).toContain("candidate had 2 ungraded trials"); expect(ungraded).not.toContain("Candidate minus baseline"); + + const infra = renderMarkdown("test", [ + { + ...summary, + arm: "candidate", + infraErrors: 1, + complete: false, + incompleteReasons: ["had 1 infrastructure failure"], + }, + { ...summary, arm: "baseline" }, + ]); + expect(infra).toContain("candidate had 1 infrastructure failure"); + expect(infra).not.toContain("Candidate minus baseline"); }); test("keeps full errors until redaction and clamps derived scores", () => { @@ -385,6 +438,7 @@ describe("benchmark workflow hardening", () => { expect(workflow).toContain("github.rest.repos.compareCommits"); expect(workflow).not.toContain("baseSha = pull.base.sha"); expect(workflow).toContain('HARBOR_VERSION: "0.21.0"'); + expect(workflow).toContain('HARBOR_HYPEMAN_VERSION: "0.1.2"'); expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"'); expect( workflow.match(/c7feaa2435ca8115c0762c44e13885fe5adf3e98/g), @@ -393,7 +447,7 @@ describe("benchmark workflow hardening", () => { expect(workflow).not.toContain( "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}", ); - expect(workflow).toContain("all(.arms[]; .scored > 0)"); + expect(workflow).toContain("all(.arms[]; .complete == true)"); expect(workflow).toMatch( /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/, ); @@ -403,6 +457,32 @@ describe("benchmark workflow hardening", () => { expect(workflow).toContain( 'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")', ); + expect(workflow).toContain('KERNEL_MCP_BENCHMARK_SOURCE_ROOT="$checkout"'); + expect(workflow).toContain( + '"$GITHUB_WORKSPACE/harness/benchmarks/harbor/clawbench/run.sh"', + ); + + const runner = readFileSync( + join(process.cwd(), "benchmarks/harbor/clawbench/run.sh"), + "utf8", + ); + expect(runner).toContain( + "source_root=${KERNEL_MCP_BENCHMARK_SOURCE_ROOT:-$harness_root}", + ); + expect(runner).toContain( + "harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.2}", + ); + expect(runner).toContain('--max-retries "${HARBOR_MAX_RETRIES:-5}"'); + for (const exception of [ + "APITimeoutError", + "APIConnectionError", + "RateLimitError", + "InternalServerError", + "ConnectionRefusedError", + "ExecProtocolError", + ]) { + expect(runner).toContain(`--retry-include ${exception}`); + } }); test("requires the benchmark credential to resolve to one project", () => { diff --git a/benchmarks/harbor/results.ts b/benchmarks/harbor/results.ts index dc24bab3..41def571 100644 --- a/benchmarks/harbor/results.ts +++ b/benchmarks/harbor/results.ts @@ -66,7 +66,10 @@ export interface ArmSummary { strict?: number; strictScored: number; infraErrors: number; + retries: number; ungraded: number; + complete: boolean; + incompleteReasons: string[]; kernelMcpValid?: number; kernelMcpChecked: number; medianCalls?: number; @@ -353,6 +356,26 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { }); const strict = numeric("reward_strict"); const validity = numeric("kernel_mcp_valid"); + const infraErrors = arm.trials.filter( + (trial) => trial.errorClass === "infra", + ).length; + const ungraded = arm.trials.filter( + (trial) => trial.errorClass !== "infra" && trial.scores.ungraded_rate === 1, + ).length; + const incompleteReasons = [ + ...(arm.nTotalTrials === 0 ? ["had no intended trials"] : []), + ...(primary.length !== arm.nTotalTrials + ? [`scored ${primary.length}/${arm.nTotalTrials} trials`] + : []), + ...(infraErrors > 0 + ? [ + `had ${infraErrors} infrastructure ${infraErrors === 1 ? "failure" : "failures"}`, + ] + : []), + ...(ungraded > 0 + ? [`had ${ungraded} ungraded ${ungraded === 1 ? "trial" : "trials"}`] + : []), + ]; const costs = arm.trials.flatMap((trial) => trial.metrics.costUsd === undefined ? [] : [trial.metrics.costUsd], ); @@ -382,12 +405,11 @@ export function summarizeArm(arm: BenchmarkArm): ArmSummary { 0, ), strictScored: strict.length, - infraErrors: arm.trials.filter((trial) => trial.errorClass === "infra") - .length, - ungraded: arm.trials.filter( - (trial) => - trial.errorClass !== "infra" && trial.scores.ungraded_rate === 1, - ).length, + infraErrors, + retries: arm.nRetries, + ungraded, + complete: incompleteReasons.length === 0, + incompleteReasons, kernelMcpValid: validity.length === 0 ? undefined