Retry transient benchmark infrastructure failures

.github/workflows/benchmark-clawbench.yml

@@ -220,7 +220,7 @@ jobs:
       BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
       BRAINTRUST_PROJECT: ${{ vars.BRAINTRUST_PROJECT }}
       HARBOR_VERSION: "0.21.0"
-      HARBOR_HYPEMAN_VERSION: "0.1.1"
+      HARBOR_HYPEMAN_VERSION: "0.1.2"
       CODEX_BENCHMARK_MODEL: gpt-5.6-luna
       CODEX_BENCHMARK_VERSION: "0.120.0"
       CLAUDE_BENCHMARK_MODEL: claude-sonnet-5
@@ -342,7 +342,8 @@ jobs:
           run_arm() {
             local checkout=$1 arm=$2 job_name=$3
             set +e
-            "$checkout/benchmarks/harbor/clawbench/run.sh" \
+            KERNEL_MCP_BENCHMARK_SOURCE_ROOT="$checkout" \
+              "$GITHUB_WORKSPACE/harness/benchmarks/harbor/clawbench/run.sh" \
               "$BENCHMARK_AGENT" \
               "$BENCHMARK_TASK" \
               "$job_name" \
@@ -479,4 +480,4 @@ jobs:
           [[ "$BASELINE_STATUS" == "0" ]]
           [[ "$PUBLISH_OUTCOME" == "success" ]]
           [[ "$REPORT_OUTCOME" == "success" ]]
-          jq -e 'all(.arms[]; .scored > 0)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null
+          jq -e 'all(.arms[]; .complete == true)' "$RUNNER_TEMP/benchmark-summary.json" >/dev/null

benchmarks/harbor/README.md

@@ -16,7 +16,7 @@ The image records the current Git SHA, and the generated task records the ClawBe
 
 ## Requirements
 
-- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.1
+- `uv`, Harbor 0.21.0, and `harbor-hypeman` 0.1.2
 - Hypeman CLI credentials
 - a ClawBench checkout containing pinned commit `c7feaa2`
 - `KERNEL_MCP_BENCHMARK_API_KEY` scoped to an isolated evaluation project; its credential scope is the project source of truth
@@ -57,6 +57,8 @@ Codex defaults to version `0.120.0` with `gpt-5.6-luna`. Claude Code defaults to
 
 Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to six hours. Set `HARBOR_BENCHMARK_TIMEOUT` to override either limit. Set `HARBOR_JOBS_DIR` to choose where Harbor writes results.
 
+The runner retries a whole isolated trial up to five times for transient Hypeman connection, timeout, and exec-stream failures. Set `HARBOR_MAX_RETRIES` to override that limit. Per-request SDK retries remain disabled because transparently retrying instance or image creation can duplicate a request whose first response was lost.
+
 ## GitHub Actions
 
 The `Benchmark ClawBench` workflow runs the complete suite weekly and on demand. Select it from the Actions tab and provide either a same-repository PR number or a ref. Comparison runs benchmark the candidate SHA against its merge base so unrelated changes on the target branch do not affect the delta. Harbor, Hypeman, agent, and model versions are pinned by the workflow and each arm's observed agent configuration appears in the report.
@@ -101,3 +103,5 @@ BRAINTRUST_PROJECT=kernel-mcp-server-benchmarks \
 ```
 
 The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Re-publication replaces the rows and refreshes experiment metadata. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments.
+
+Reports suppress comparison deltas when either arm has an infrastructure failure or ungraded trial. The workflow fails unless every intended trial is graded, while still retaining the incomplete report for diagnosis.

benchmarks/harbor/clawbench/run.sh

@@ -15,10 +15,11 @@ agent=${1:-}
 [[ "$agent" == "claude-code" || "$agent" == "codex" ]] || usage
 task_id=${2:-v2-1134-chapter-finder-redcross}
 
-repo_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
-benchmark_dir="$repo_root/benchmarks/harbor"
-image_env="$benchmark_dir/.image.env"
-clawbench_repo=${CLAWBENCH_REPO:-$repo_root/../ClawBench}
+harness_root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
+source_root=${KERNEL_MCP_BENCHMARK_SOURCE_ROOT:-$harness_root}
+benchmark_dir="$harness_root/benchmarks/harbor"
+image_env="$source_root/benchmarks/harbor/.image.env"
+clawbench_repo=${CLAWBENCH_REPO:-$harness_root/../ClawBench}
 clawbench_ref=${CLAWBENCH_REF:-c7feaa2435ca8115c0762c44e13885fe5adf3e98}
 
 [[ -f "$image_env" ]] || {
@@ -69,7 +70,7 @@ case "$agent" in
 esac
 
 harbor_version=${HARBOR_VERSION:-0.21.0}
-harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.1}
+harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.2}
 
 export KERNEL_API_KEY=$KERNEL_MCP_BENCHMARK_API_KEY
 export KERNEL_BASE_URL=${KERNEL_BASE_URL:-https://api.onkernel.com}
@@ -161,6 +162,12 @@ timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_time
   --job-name "$job_name" \
   --jobs-dir "$jobs_dir" \
   --n-concurrent "${HARBOR_N_CONCURRENT:-1}" \
-  --max-retries 0 \
+  --max-retries "${HARBOR_MAX_RETRIES:-5}" \
+  --retry-include APITimeoutError \
+  --retry-include APIConnectionError \
+  --retry-include RateLimitError \
+  --retry-include InternalServerError \
+  --retry-include ConnectionRefusedError \
+  --retry-include ExecProtocolError \
   --delete \
   --yes

benchmarks/harbor/publish-braintrust.ts

@@ -354,13 +354,17 @@ export async function publishBenchmark(
   experimentName: string,
   apiKey: string,
 ): Promise<Record<string, unknown>> {
-  const ungraded = arms
+  const incomplete = arms
     .map(summarizeArm)
-    .filter((summary) => summary.scored === 0)
-    .map((summary) => summary.arm);
-  if (ungraded.length > 0) {
+    .filter((summary) => !summary.complete);
+  if (incomplete.length > 0) {
     throw new Error(
-      `Cannot publish benchmark without graded trials for: ${ungraded.join(", ")}`,
+      `Cannot publish incomplete benchmark arms: ${incomplete
+        .map(
+          (summary) =>
+            `${summary.arm} (${summary.incompleteReasons.join(", ")})`,
+        )
+        .join("; ")}`,
     );
   }
 

benchmarks/harbor/report.ts

@@ -89,12 +89,14 @@ export function renderMarkdown(
 ): string {
   const lines = ["<!-- kernel-mcp-clawbench -->", `## ${title}`];
   const failed = Object.entries(statuses).filter(([, status]) => status !== 0);
-  const ungraded = summaries.filter((summary) => summary.scored === 0);
-  const incomplete = failed.length > 0 || ungraded.length > 0;
+  const incomplete =
+    failed.length > 0 || summaries.some((summary) => !summary.complete);
   if (incomplete) {
     const reasons = [
       ...failed.map(([arm, status]) => `${arm} exited ${status}`),
-      ...ungraded.map((summary) => `${summary.arm} produced no graded trials`),
+      ...summaries.flatMap((summary) =>
+        summary.incompleteReasons.map((reason) => `${summary.arm} ${reason}`),
+      ),
     ];
     lines.push(
       "",
@@ -103,12 +105,12 @@ export function renderMarkdown(
   }
   lines.push(
     "",
-    "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |",
-    "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
+    "| Arm | Configuration | Lenient | Strict | Intercepted | Infra | Retries | Ungraded | Kernel MCP valid | Median calls | Median duration | Cost |",
+    "|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|",
   );
   for (const summary of summaries) {
     lines.push(
-      `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`,
+      `| ${summary.arm} | ${summary.configuration ?? "—"} | ${ratio(summary.lenient, summary.trials)} | ${ratio(summary.strict, summary.trials)} | ${ratio(summary.intercepted, summary.trials)} | ${summary.infraErrors} | ${summary.retries} | ${summary.ungraded} | ${ratio(summary.kernelMcpValid, summary.kernelMcpChecked)} | ${summary.medianCalls ?? "—"} | ${duration(summary.medianDurationMs)} | ${cost(summary.totalCostUsd)} |`,
     );
   }
 

benchmarks/harbor/results.test.ts

@@ -114,6 +114,24 @@ function fixture(): string {
   return root;
 }
 
+function completeArm() {
+  const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
+  const failed = arm.trials[1];
+  failed.error = undefined;
+  failed.errorClass = undefined;
+  failed.rewards = { reward: 0, intercepted: 0 };
+  failed.scores = {
+    accuracy: 0,
+    false_positive_rate: 0,
+    false_negative_rate: 1,
+    infra_error_rate: 0,
+    intercepted: 0,
+    reward: 0,
+    ungraded_rate: 0,
+  };
+  return arm;
+}
+
 describe("Harbor result ingestion", () => {
   test("keeps infrastructure errors out of task-quality scores", () => {
     const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
@@ -146,13 +164,35 @@ describe("Harbor result ingestion", () => {
       strict: 0,
       intercepted: 1,
       infraErrors: 1,
+      retries: 0,
       ungraded: 0,
+      complete: false,
+      incompleteReasons: ["scored 1/2 trials", "had 1 infrastructure failure"],
       kernelMcpValid: 1,
       medianCalls: 1,
       totalCostUsd: 0.01,
     });
   });
 
+  test("requires every intended trial to be graded", () => {
+    expect(summarizeArm(completeArm()).complete).toBe(true);
+
+    const missing = completeArm();
+    missing.nTotalTrials = 3;
+    expect(summarizeArm(missing)).toMatchObject({
+      complete: false,
+      incompleteReasons: ["scored 2/3 trials"],
+    });
+
+    const empty = completeArm();
+    empty.nTotalTrials = 0;
+    empty.trials = [];
+    expect(summarizeArm(empty)).toMatchObject({
+      complete: false,
+      incompleteReasons: ["had no intended trials"],
+    });
+  });
+
   test("builds deterministic root, llm, and tool spans", () => {
     const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
     const first = buildExperimentEvents([arm], "test-experiment");
@@ -192,7 +232,7 @@ describe("Harbor result ingestion", () => {
   });
 
   test("re-publishes the same rows and spans by deterministic ID", async () => {
-    const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
+    const arm = completeArm();
     const originalFetch = globalThis.fetch;
     const inserts: string[][] = [];
     const metadataUpdates: unknown[] = [];
@@ -258,34 +298,20 @@ describe("Harbor result ingestion", () => {
     }
   });
 
-  test("does not publish arms without graded trials", async () => {
+  test("does not publish incomplete arms", async () => {
     const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
     for (const trial of arm.trials) trial.rewards = {};
     await expect(
       publishBenchmark([arm], "project", "experiment", "api-key"),
-    ).rejects.toThrow(
-      "Cannot publish benchmark without graded trials for: candidate",
-    );
+    ).rejects.toThrow("Cannot publish incomplete benchmark arms: candidate");
   });
 
   test("uses the lenient reward per trial and reports incomplete arms", () => {
     expect(selectPrimaryReward({ reward: 0, reward_lenient: 1 })).toEqual({
       key: "reward_lenient",
       value: 1,
     });
-    const arm = readBenchmarkArm({ name: "candidate", path: fixture() });
-    const second = arm.trials[1];
-    second.error = undefined;
-    second.errorClass = undefined;
-    second.rewards = { reward: 0 };
-    second.scores = {
-      accuracy: 0,
-      false_positive_rate: 0,
-      false_negative_rate: 1,
-      infra_error_rate: 0,
-      reward: 0,
-      ungraded_rate: 0,
-    };
+    const arm = completeArm();
     const summary = summarizeArm(arm);
     expect(summary.scored).toBe(2);
     expect(summary.lenient).toBe(1);
@@ -300,11 +326,38 @@ describe("Harbor result ingestion", () => {
       ]),
     ).toContain("+0.1 lenient");
     const ungraded = renderMarkdown("test", [
-      { ...summary, arm: "candidate", scored: 0, ungraded: summary.trials },
-      { ...summary, arm: "baseline", scored: 0, ungraded: summary.trials },
+      {
+        ...summary,
+        arm: "candidate",
+        scored: 0,
+        ungraded: summary.trials,
+        complete: false,
+        incompleteReasons: ["had 2 ungraded trials"],
+      },
+      {
+        ...summary,
+        arm: "baseline",
+        scored: 0,
+        ungraded: summary.trials,
+        complete: false,
+        incompleteReasons: ["had 2 ungraded trials"],
+      },
     ]);
-    expect(ungraded).toContain("candidate produced no graded trials");
+    expect(ungraded).toContain("candidate had 2 ungraded trials");
     expect(ungraded).not.toContain("Candidate minus baseline");
+
+    const infra = renderMarkdown("test", [
+      {
+        ...summary,
+        arm: "candidate",
+        infraErrors: 1,
+        complete: false,
+        incompleteReasons: ["had 1 infrastructure failure"],
+      },
+      { ...summary, arm: "baseline" },
+    ]);
+    expect(infra).toContain("candidate had 1 infrastructure failure");
+    expect(infra).not.toContain("Candidate minus baseline");
   });
 
   test("keeps full errors until redaction and clamps derived scores", () => {
@@ -385,6 +438,7 @@ describe("benchmark workflow hardening", () => {
     expect(workflow).toContain("github.rest.repos.compareCommits");
     expect(workflow).not.toContain("baseSha = pull.base.sha");
     expect(workflow).toContain('HARBOR_VERSION: "0.21.0"');
+    expect(workflow).toContain('HARBOR_HYPEMAN_VERSION: "0.1.2"');
     expect(workflow).toContain('CODEX_BENCHMARK_VERSION: "0.120.0"');
     expect(
       workflow.match(/c7feaa2435ca8115c0762c44e13885fe5adf3e98/g),
@@ -393,7 +447,7 @@ describe("benchmark workflow hardening", () => {
     expect(workflow).not.toContain(
       "KERNEL_PROJECT: ${{ vars.KERNEL_PROJECT }}",
     );
-    expect(workflow).toContain("all(.arms[]; .scored > 0)");
+    expect(workflow).toContain("all(.arms[]; .complete == true)");
     expect(workflow).toMatch(
       /- name: Mark the PR benchmark as running\n\s+if:.*\n\s+continue-on-error: true/,
     );
@@ -403,6 +457,32 @@ describe("benchmark workflow hardening", () => {
     expect(workflow).toContain(
       'statuses=(--status "candidate=${CANDIDATE_STATUS:-1}")',
     );
+    expect(workflow).toContain('KERNEL_MCP_BENCHMARK_SOURCE_ROOT="$checkout"');
+    expect(workflow).toContain(
+      '"$GITHUB_WORKSPACE/harness/benchmarks/harbor/clawbench/run.sh"',
+    );
+
+    const runner = readFileSync(
+      join(process.cwd(), "benchmarks/harbor/clawbench/run.sh"),
+      "utf8",
+    );
+    expect(runner).toContain(
+      "source_root=${KERNEL_MCP_BENCHMARK_SOURCE_ROOT:-$harness_root}",
+    );
+    expect(runner).toContain(
+      "harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.2}",
+    );
+    expect(runner).toContain('--max-retries "${HARBOR_MAX_RETRIES:-5}"');
+    for (const exception of [
+      "APITimeoutError",
+      "APIConnectionError",
+      "RateLimitError",
+      "InternalServerError",
+      "ConnectionRefusedError",
+      "ExecProtocolError",
+    ]) {
+      expect(runner).toContain(`--retry-include ${exception}`);
+    }
   });
 
   test("requires the benchmark credential to resolve to one project", () => {