diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index 7b7b2ce..0e3d0bb 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -57,7 +57,9 @@ Codex defaults to version `0.120.0` with `gpt-5.6-luna`. Claude Code defaults to Single-task runs have a 40-minute wall-clock limit. Full-suite runs default to six hours. Set `HARBOR_BENCHMARK_TIMEOUT` to override either limit. Set `HARBOR_JOBS_DIR` to choose where Harbor writes results. -The runner retries a whole isolated trial up to five times for transient Hypeman connection, timeout, and exec-stream failures. Set `HARBOR_MAX_RETRIES` to override that limit. Per-request SDK retries remain disabled because transparently retrying instance or image creation can duplicate a request whose first response was lost. +Before creating the Harbor dataset, the runner creates and deletes one disposable PurelyMail account. API errors stop the run before trials begin, so missing email accounts cannot silently become task failures. + +The runner retries a whole isolated trial up to five times for transient Hypeman connection, timeout, exec-stream, and agent/task setup failures. Set `HARBOR_MAX_RETRIES` to override that limit. Per-request SDK retries remain disabled because transparently retrying instance or image creation can duplicate a request whose first response was lost. ## GitHub Actions @@ -102,6 +104,8 @@ BRAINTRUST_PROJECT=kernel-mcp-server-benchmarks \ --arm baseline=/path/to/baseline-job ``` -The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Re-publication replaces the rows and refreshes experiment metadata. Rows contain task identity, numeric rewards, provenance, bounded errors, timing, token, call, and cost metrics. ATIF agent/tool activity is attached as child spans after secret redaction. Task instructions, ground truth, browser session URLs, and recordings are not placed on experiment rows or public pull-request comments. +The experiment name and deterministic row/span IDs make it safe to publish the same job directories again. Re-publication replaces the rows and refreshes experiment metadata. Rows contain task identity, the redacted instruction, numeric rewards, provenance, bounded errors, and trial timing. Setup, browser lifetime, execution, verification, and finalization are separate timeline spans; the browser span records its timeout and deletion status without its identifiers. ATIF turns include their preceding context, structured tool calls, cached-token metrics, and inferred turn intervals. Browser tool spans record whether the supplied session ID matched the trial's expected session without publishing either ID. Tool durations remain unspecified because ATIF does not record them. + +The publisher redacts typed form values, configured secrets, credentials, email addresses, browser session/replay IDs, private-info file contents, and provider URLs. It validates the final payload and aborts before creating an experiment if sensitive content remains. Ground truth, recordings, and raw Harbor jobs are never published. Reports suppress comparison deltas when either arm has an infrastructure failure or ungraded trial. The workflow fails unless every intended trial is graded, while still retaining the incomplete report for diagnosis. diff --git a/benchmarks/harbor/clawbench/run.sh b/benchmarks/harbor/clawbench/run.sh index 1feda7e..04e8417 100755 --- a/benchmarks/harbor/clawbench/run.sh +++ b/benchmarks/harbor/clawbench/run.sh @@ -48,6 +48,8 @@ set +a : "${PURELY_MAIL_API_KEY:?PURELY_MAIL_API_KEY is required}" : "${PURELY_MAIL_DOMAIN:?PURELY_MAIL_DOMAIN is required}" +bun "$benchmark_dir/verify-purelymail.ts" + case "$agent" in claude-code) if [[ -z "${ANTHROPIC_API_KEY:-}" && -z "${ANTHROPIC_AUTH_TOKEN:-}" && -z "${CLAUDE_CODE_OAUTH_TOKEN:-}" ]]; then @@ -169,5 +171,7 @@ timeout --signal=INT --kill-after=30s "${HARBOR_BENCHMARK_TIMEOUT:-$default_time --retry-include InternalServerError \ --retry-include ConnectionRefusedError \ --retry-include ExecProtocolError \ + --retry-include AgentSetupTimeoutError \ + --retry-include RuntimeError \ --delete \ --yes diff --git a/benchmarks/harbor/publish-braintrust.ts b/benchmarks/harbor/publish-braintrust.ts index c6fa1c1..4a8ad0d 100644 --- a/benchmarks/harbor/publish-braintrust.ts +++ b/benchmarks/harbor/publish-braintrust.ts @@ -4,13 +4,22 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname } from "node:path"; import { type BenchmarkArm, + type BenchmarkPhase, type BenchmarkTrial, parseArmSpec, readBenchmarkArm, selectPrimaryReward, summarizeArm, } from "./results"; -import { redactString, redactValue } from "./redact"; +import { + assertSafeToPublish, + collectSensitiveValues, + privateInfoRead, + redactString, + redactValue, + redactValueWithSecrets, + REDACTED_PRIVATE_INFO, +} from "./redact"; interface CliOptions { arms: string[]; @@ -41,7 +50,7 @@ interface BraintrustEvent { span_id: string; root_span_id: string; span_parents: string[]; - span_attributes: { name: string; type: "eval" | "llm" | "tool" }; + span_attributes: { name: string; type: "eval" | "llm" | "tool" | "task" }; created?: string; input?: unknown; output?: unknown; @@ -127,16 +136,19 @@ function metricRecord(trial: BenchmarkTrial): Record { Object.entries({ start: trial.metrics.start, end: trial.metrics.end, - input_tokens: trial.metrics.inputTokens, - cached_tokens: trial.metrics.cacheTokens, - output_tokens: trial.metrics.outputTokens, - cost_usd: trial.metrics.costUsd, duration_ms: trial.metrics.durationMs, tool_calls: trial.metrics.toolCalls, }).filter((entry): entry is [string, number] => entry[1] !== undefined), ); } +function taskInstruction(trial: BenchmarkTrial): unknown { + const userSteps = trajectorySteps(trial).filter( + (step) => step.source === "user" && step.message !== undefined, + ); + return redactValue(userSteps.at(-1)?.message); +} + function trialMetadata(trial: BenchmarkTrial): Record { return { trialName: trial.trialName, @@ -161,24 +173,220 @@ function trialMetadata(trial: BenchmarkTrial): Record { }; } +function timestampSeconds(value?: string): number | undefined { + if (!value) return undefined; + const millis = Date.parse(value); + return Number.isFinite(millis) ? millis / 1000 : undefined; +} + +function timestampIso(value?: number): string | undefined { + return value === undefined ? undefined : new Date(value * 1000).toISOString(); +} + +function stepContext( + step: AtifStep, + sensitiveValues: string[], +): Record { + const calls = step.tool_calls ?? []; + return { + source: step.source, + message: redactValueWithSecrets(step.message, sensitiveValues), + toolCalls: calls.map((call) => ({ + name: call.function_name ?? "tool", + arguments: redactValueWithSecrets(call.arguments, sensitiveValues), + })), + observations: (step.observation?.results ?? []).map((result) => { + const call = calls.find( + (candidate) => candidate.tool_call_id === result.source_call_id, + ); + const toolName = call?.function_name ?? "tool"; + return { + source_call_id: result.source_call_id, + content: + call && privateInfoRead(toolName, call.arguments) + ? REDACTED_PRIVATE_INFO + : redactValueWithSecrets(result.content, sensitiveValues), + }; + }), + }; +} + +function llmInput( + steps: AtifStep[], + stepIndex: number, + sensitiveValues: string[], +): unknown { + const prior = steps.slice(0, stepIndex); + let previousAgent = -1; + for (let index = prior.length - 1; index >= 0; index -= 1) { + if (prior[index].source === "agent") { + previousAgent = index; + break; + } + } + const context = + previousAgent === -1 + ? prior + : prior.slice(previousAgent, previousAgent + 1); + return context.map((step) => stepContext(step, sensitiveValues)); +} + +function llmOutput(step: AtifStep, sensitiveValues: string[]): unknown { + if ((step.tool_calls ?? []).length === 0) { + return redactValueWithSecrets(step.message, sensitiveValues); + } + return { + message: redactValueWithSecrets(step.message, sensitiveValues), + toolCalls: (step.tool_calls ?? []).map((call) => ({ + name: call.function_name ?? "tool", + arguments: redactValueWithSecrets(call.arguments, sensitiveValues), + })), + }; +} + +function privateInfoValues(steps: AtifStep[]): string[] { + const values = new Set(); + for (const step of steps) { + for (const call of step.tool_calls ?? []) { + const toolName = call.function_name ?? "tool"; + if (!privateInfoRead(toolName, call.arguments)) continue; + const observation = step.observation?.results?.find( + (result) => result.source_call_id === call.tool_call_id, + ); + if (typeof observation?.content !== "string") continue; + for (const match of observation.content.matchAll( + /["']\s*:\s*["']([^"'\\]{4,})["']/g, + )) { + values.add(match[1]); + } + } + } + return [...values]; +} + +function phaseEvent( + rowId: string, + name: string, + phase: BenchmarkPhase, + metadata: Record = {}, +): BraintrustEvent | undefined { + if (phase.start === undefined || phase.end === undefined) return undefined; + const id = uuidV5(`${rowId}:phase:${name}`); + return { + id, + span_id: id, + root_span_id: rowId, + span_parents: [rowId], + span_attributes: { name, type: "task" }, + created: timestampIso(phase.start), + metadata: { phase: name, ...metadata }, + metrics: { + start: phase.start, + end: phase.end, + ...(phase.durationMs === undefined + ? {} + : { duration_ms: phase.durationMs }), + }, + _is_merge: false, + }; +} + +function phaseEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { + const events: BraintrustEvent[] = []; + for (const [name, phase] of Object.entries({ + environment_setup: trial.phases.environmentSetup, + agent_setup: trial.phases.agentSetup, + agent_execution: trial.phases.agentExecution, + verifier: trial.phases.verifier, + })) { + if (!phase) continue; + const event = phaseEvent(rowId, name, phase); + if (event) events.push(event); + } + + const stepSetupStart = trial.phases.agentSetup?.end; + const stepSetupEnd = trial.phases.agentExecution?.start; + if ( + stepSetupStart !== undefined && + stepSetupEnd !== undefined && + stepSetupEnd > stepSetupStart + ) { + const event = phaseEvent(rowId, "step_setup", { + start: stepSetupStart, + end: stepSetupEnd, + durationMs: (stepSetupEnd - stepSetupStart) * 1000, + }); + if (event) events.push(event); + } + + if (trial.browser?.start !== undefined && trial.browser.end !== undefined) { + const event = phaseEvent( + rowId, + "browser_session", + { + start: trial.browser.start, + end: trial.browser.end, + durationMs: (trial.browser.end - trial.browser.start) * 1000, + }, + { + timeoutSeconds: trial.browser.timeoutSeconds, + deletionVerified: trial.browser.deletionVerified, + }, + ); + if (event) events.push(event); + } + + const finalizeStart = trial.phases.verifier?.end; + const finalizeEnd = trial.metrics.end; + if ( + finalizeStart !== undefined && + finalizeEnd !== undefined && + finalizeEnd > finalizeStart + ) { + const event = phaseEvent(rowId, "finalize", { + start: finalizeStart, + end: finalizeEnd, + durationMs: (finalizeEnd - finalizeStart) * 1000, + }); + if (event) events.push(event); + } + return events; +} + function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { const events: BraintrustEvent[] = []; - const agentSteps = trajectorySteps(trial).filter( - (candidate) => candidate.source === "agent", - ); - for (const [stepIndex, step] of agentSteps.entries()) { + const steps = trajectorySteps(trial); + const sensitiveValues = [ + ...collectSensitiveValues(steps), + ...privateInfoValues(steps), + ]; + const agentExecutionId = uuidV5(`${rowId}:phase:agent_execution`); + let previousEnd = trial.phases.agentExecution?.start; + + for (const [trajectoryIndex, step] of steps.entries()) { + if (step.source !== "agent") continue; const stepId = step.step_id; - const stepKey = `${stepId ?? "missing"}:${stepIndex}`; + const stepKey = `${stepId ?? "missing"}:${trajectoryIndex}`; const llmId = uuidV5(`${rowId}:llm:${stepKey}`); - const start = step.timestamp - ? Date.parse(step.timestamp) / 1000 - : undefined; + const end = timestampSeconds(step.timestamp); + const start = + previousEnd === undefined || end === undefined + ? end + : Math.min(previousEnd, end); + if (end !== undefined) previousEnd = end; + const promptTokens = number(step.metrics?.prompt_tokens); + const completionTokens = number(step.metrics?.completion_tokens); const llmMetrics = Object.fromEntries( Object.entries({ start, - end: start, - prompt_tokens: number(step.metrics?.prompt_tokens), - completion_tokens: number(step.metrics?.completion_tokens), + end, + prompt_tokens: promptTokens, + prompt_cached_tokens: number(step.metrics?.cached_tokens), + completion_tokens: completionTokens, + tokens: + promptTokens === undefined || completionTokens === undefined + ? undefined + : promptTokens + completionTokens, cost_usd: number(step.metrics?.cost_usd), }).filter((entry): entry is [string, number] => entry[1] !== undefined), ); @@ -186,14 +394,21 @@ function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { id: llmId, span_id: llmId, root_span_id: rowId, - span_parents: [rowId], + span_parents: [ + trial.phases.agentExecution?.start !== undefined && + trial.phases.agentExecution.end !== undefined + ? agentExecutionId + : rowId, + ], span_attributes: { name: "agent", type: "llm" }, - created: step.timestamp, - output: redactValue(step.message), + created: timestampIso(start) ?? step.timestamp, + input: llmInput(steps, trajectoryIndex, sensitiveValues), + output: llmOutput(step, sensitiveValues), metadata: { phase: "agent_execution", + timing: "ATIF turn completion interval", stepId, - stepIndex, + trajectoryIndex, model: step.model_name ?? trial.model, }, metrics: llmMetrics, @@ -207,22 +422,34 @@ function atifEvents(trial: BenchmarkTrial, rowId: string): BraintrustEvent[] { const observation = step.observation?.results?.find( (result) => result.source_call_id === call.tool_call_id, ); + const toolName = call.function_name ?? "tool"; + const sessionId = + call.arguments !== null && typeof call.arguments === "object" + ? (call.arguments as Record).session_id + : undefined; + const sessionIdMatchesExpected = + typeof sessionId === "string" && trial.expectedBrowserSessionId + ? sessionId === trial.expectedBrowserSessionId + : undefined; events.push({ id: toolId, span_id: toolId, root_span_id: rowId, span_parents: [llmId], - span_attributes: { name: call.function_name ?? "tool", type: "tool" }, + span_attributes: { name: toolName, type: "tool" }, created: step.timestamp, - input: redactValue(call.arguments), - output: redactValue(observation?.content), + input: redactValueWithSecrets(call.arguments, sensitiveValues), + output: privateInfoRead(toolName, call.arguments) + ? REDACTED_PRIVATE_INFO + : redactValueWithSecrets(observation?.content, sensitiveValues), metadata: { phase: "agent_execution", + timing: "not available in ATIF", stepId, - stepIndex, + trajectoryIndex, toolCallId: call.tool_call_id, + sessionIdMatchesExpected, }, - metrics: start === undefined ? undefined : { start: start, end: start }, _is_merge: false, }); } @@ -249,7 +476,11 @@ export function buildExperimentEvents( span_parents: [], span_attributes: { name: trial.taskName, type: "eval" }, created: trial.startedAt, - input: { source: trial.source, taskName: trial.taskName }, + input: { + source: trial.source, + taskName: trial.taskName, + instruction: taskInstruction(trial), + }, output: { reward: primaryReward?.value, rewardKey: primaryReward?.key, @@ -262,9 +493,11 @@ export function buildExperimentEvents( metrics: metricRecord(trial), _is_merge: false, }); + events.push(...phaseEvents(trial, rowId)); events.push(...atifEvents(trial, rowId)); } } + assertSafeToPublish(events); return events; } diff --git a/benchmarks/harbor/redact.ts b/benchmarks/harbor/redact.ts index 851c792..06f73a2 100644 --- a/benchmarks/harbor/redact.ts +++ b/benchmarks/harbor/redact.ts @@ -1,7 +1,46 @@ const SECRET_NAME = /(API_KEY|TOKEN|SECRET|PASSWORD|PRIVATE_KEY|CREDENTIAL)/i; -const SENSITIVE_FIELD = - /(API_KEY|TOKEN|JWT|SECRET|PASSWORD|PRIVATE_KEY|CREDENTIAL|^COOKIE$|^SET-COOKIE$)/i; const REDACTED = "[REDACTED]"; +const REDACTED_PRIVATE_INFO = "[REDACTED_PRIVATE_INFO]"; + +const SENSITIVE_FIELDS = new Set([ + "api_key", + "access_token", + "auth_token", + "refresh_token", + "session_token", + "token", + "jwt", + "secret", + "password", + "private_key", + "credential", + "credentials", + "cookie", + "set-cookie", + "session_id", + "replay_id", + "cdp_url", + "cdp_ws_url", + "viewer_url", + "browser_live_view_url", + "email", + "phone", + "address", +]); + +function normalizedField(key: string): string { + return key.trim().toLowerCase().replace(/-/g, "_"); +} + +function sensitiveField(key: string): boolean { + const normalized = normalizedField(key); + return ( + SENSITIVE_FIELDS.has(normalized) || + /(?:^|_)(?:api_key|access_token|auth_token|refresh_token|session_token|password|private_key|credential|session_id|replay_id|cdp_url|viewer_url)$/.test( + normalized, + ) + ); +} function secretValues(): string[] { return Object.entries(process.env) @@ -12,22 +51,52 @@ function secretValues(): string[] { .sort((left, right) => right.length - left.length); } -export function redactString(value: string, maxLength = 20_000): string { +const TYPED_CALL = /\.(?:fill|type)\(([^)]*)\)/g; +const STRING_LITERAL = /(["'`])((?:\\.|(?!\1).)*)\1/g; + +function typedCallValues(value: string): string[] { + const values: string[] = []; + for (const call of value.matchAll(TYPED_CALL)) { + const literals = [...call[1].matchAll(STRING_LITERAL)]; + const typedValue = literals.at(-1)?.[2]; + if (typedValue !== undefined) values.push(typedValue); + } + return values; +} + +function redactTypedLiterals(value: string): string { + return value.replace(TYPED_CALL, (call, argumentsText: string) => { + const literals = [...argumentsText.matchAll(STRING_LITERAL)]; + const typedValue = literals.at(-1); + if (!typedValue || typedValue.index === undefined) return call; + const start = typedValue.index; + const end = start + typedValue[0].length; + const quote = typedValue[1]; + const redactedArguments = `${argumentsText.slice(0, start)}${quote}${REDACTED}${quote}${argumentsText.slice(end)}`; + return call.replace(argumentsText, redactedArguments); + }); +} + +export function redactStringWithSecrets( + value: string, + additionalSecrets: string[], + maxLength = 20_000, +): string { let redacted = value; - for (const secret of secretValues()) { - redacted = redacted.split(secret).join(REDACTED); + for (const secret of [...secretValues(), ...additionalSecrets]) { + if (secret.length >= 4) redacted = redacted.split(secret).join(REDACTED); } - redacted = redacted + redacted = redactTypedLiterals(redacted) .replace(/\bBearer\s+[A-Za-z0-9._~+/=-]+/gi, `Bearer ${REDACTED}`) .replace(/\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b/g, REDACTED) .replace(/\b(?:sk|pk|bt|kapi|whsec)[-_][A-Za-z0-9_-]{12,}\b/gi, REDACTED) .replace( - /(["']?(?:api[_-]?key|access[_-]?token|credential|jwt|password|secret|token)["']?\s*[:=]\s*["']?)[^"'\s,}&]+/gi, + /(["']?(?:api[_-]?key|access[_-]?token|auth[_-]?token|credential|jwt|password|private[_-]?key|refresh[_-]?token|replay[_-]?id|secret|session[_-]?id|session[_-]?token|token)["']?\s*[:=]\s*["']?)[^"'\s,}&]+/gi, `$1${REDACTED}`, ) .replace( - /([?&](?:api[_-]?key|access[_-]?token|auth|code|credential|jwt|password|secret|session[_-]?token|token)=)[^&#\s]+/gi, + /([?&](?:api[_-]?key|access[_-]?token|auth|code|credential|jwt|password|secret|session[_-]?id|session[_-]?token|token)=)[^&#\s]+/gi, `$1${REDACTED}`, ) .replace(/(\b(?:cookie|set-cookie)\s*:\s*)[^\r\n]+/gi, `$1${REDACTED}`) @@ -40,20 +109,150 @@ export function redactString(value: string, maxLength = 20_000): string { : redacted; } -export function redactValue(value: unknown, maxStringLength = 20_000): unknown { - if (typeof value === "string") return redactString(value, maxStringLength); +export function redactString(value: string, maxLength = 20_000): string { + return redactStringWithSecrets(value, [], maxLength); +} + +export function redactValueWithSecrets( + value: unknown, + additionalSecrets: string[], + maxStringLength = 20_000, +): unknown { + if (typeof value === "string") { + return redactStringWithSecrets(value, additionalSecrets, maxStringLength); + } if (Array.isArray(value)) { - return value.map((entry) => redactValue(entry, maxStringLength)); + return value.map((entry) => + redactValueWithSecrets(entry, additionalSecrets, maxStringLength), + ); } if (value !== null && typeof value === "object") { return Object.fromEntries( Object.entries(value as Record).map(([key, entry]) => [ key, - SENSITIVE_FIELD.test(key) + sensitiveField(key) ? REDACTED - : redactValue(entry, maxStringLength), + : redactValueWithSecrets(entry, additionalSecrets, maxStringLength), ]), ); } return value; } + +export function redactValue(value: unknown, maxStringLength = 20_000): unknown { + return redactValueWithSecrets(value, [], maxStringLength); +} + +export function collectSensitiveValues(value: unknown): string[] { + const values = new Set(); + const collectString = (text: string) => { + for (const match of text.matchAll( + /["']?(?:password|private[_-]?key|secret|session[_-]?id|replay[_-]?id)["']?\s*[:=]\s*["']([^"'\s,}&]{4,})/gi, + )) { + values.add(match[1]); + } + for (const typedValue of typedCallValues(text)) { + if (typedValue.length >= 4) values.add(typedValue); + } + for (const match of text.matchAll( + /["'](?:id|name|type)["']\s*:\s*["'][^"']*password[^"']*["'][\s\S]{0,300}?["']value["']\s*:\s*["']([^"']{4,})["']/gi, + )) { + values.add(match[1]); + } + }; + const visit = (entry: unknown, key?: string) => { + if (typeof entry === "string") { + if (key && sensitiveField(key) && entry.length >= 4) values.add(entry); + collectString(entry); + return; + } + if (Array.isArray(entry)) { + for (const item of entry) visit(item); + return; + } + if (entry !== null && typeof entry === "object") { + for (const [childKey, child] of Object.entries( + entry as Record, + )) { + visit(child, childKey); + } + } + }; + visit(value); + return [...values].sort((left, right) => right.length - left.length); +} + +export function privateInfoRead(toolName: string, input: unknown): boolean { + if (!/(?:^|__)(?:exec_command|bash|read)$/i.test(toolName)) return false; + const fields = + input !== null && typeof input === "object" + ? (input as Record) + : {}; + const text = [ + fields.cmd, + fields.command, + fields.file_path, + fields.path, + typeof input === "string" ? input : undefined, + ] + .filter((entry) => entry !== undefined) + .map(String) + .join("\n"); + const paths = [ + ...text.matchAll( + /(?:^|[\s"'`=])(?:\.\/|\/(?:workspace\/)?|workspace\/)?my-info\/([^\s"'`;)]*)/g, + ), + ].map((match) => match[1]); + return paths.some( + (path) => + path.length === 0 || !/^kernel_browser\.json(?:$|[?#])/.test(path), + ); +} + +function assertSafeString(value: string): void { + if (/\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b/i.test(value)) { + throw new Error("Braintrust payload still contains an email address"); + } + for (const typedValue of typedCallValues(value)) { + if (typedValue !== REDACTED) { + throw new Error("Braintrust payload still contains a typed form value"); + } + } + for (const match of value.matchAll( + /["']?(?:api[_-]?key|access[_-]?token|auth[_-]?token|credential|jwt|password|private[_-]?key|refresh[_-]?token|replay[_-]?id|secret|session[_-]?id|session[_-]?token)["']?\s*[:=]\s*["']?([^"'\s,}&]+)/gi, + )) { + if (match[1] !== REDACTED) { + throw new Error( + "Braintrust payload still contains a sensitive field value", + ); + } + } + for (const secret of secretValues()) { + if (value.includes(secret)) { + throw new Error("Braintrust payload still contains a configured secret"); + } + } +} + +export function assertSafeToPublish(value: unknown): void { + if (typeof value === "string") { + assertSafeString(value); + return; + } + if (Array.isArray(value)) { + for (const entry of value) assertSafeToPublish(entry); + return; + } + if (value !== null && typeof value === "object") { + for (const [key, entry] of Object.entries( + value as Record, + )) { + if (sensitiveField(key) && entry !== REDACTED) { + throw new Error(`Braintrust payload did not redact ${key}`); + } + assertSafeToPublish(entry); + } + } +} + +export { REDACTED, REDACTED_PRIVATE_INFO }; diff --git a/benchmarks/harbor/results.test.ts b/benchmarks/harbor/results.test.ts index a077409..d259223 100644 --- a/benchmarks/harbor/results.test.ts +++ b/benchmarks/harbor/results.test.ts @@ -11,7 +11,12 @@ import { join } from "node:path"; import { buildExperimentEvents, publishBenchmark } from "./publish-braintrust"; import { renderMarkdown } from "./report"; import { readBenchmarkArm, selectPrimaryReward, summarizeArm } from "./results"; -import { redactString, redactValue } from "./redact"; +import { + assertSafeToPublish, + privateInfoRead, + redactString, + redactValue, +} from "./redact"; import { assertProjectScopedCredential } from "./verify-project-scope"; const temporaryDirectories: string[] = []; @@ -66,6 +71,14 @@ function fixture(): string { }, started_at: "2026-01-01T00:00:00Z", finished_at: "2026-01-01T00:01:00Z", + environment_setup: { + started_at: "2026-01-01T00:00:00Z", + finished_at: "2026-01-01T00:00:05Z", + }, + agent_setup: { + started_at: "2026-01-01T00:00:05Z", + finished_at: "2026-01-01T00:00:10Z", + }, step_results: [ { agent_result: { @@ -74,6 +87,14 @@ function fixture(): string { n_output_tokens: 20, cost_usd: 0.01, }, + agent_execution: { + started_at: "2026-01-01T00:00:20Z", + finished_at: "2026-01-01T00:00:40Z", + }, + verifier: { + started_at: "2026-01-01T00:00:45Z", + finished_at: "2026-01-01T00:00:55Z", + }, }, ], }); @@ -81,19 +102,40 @@ function fixture(): string { steps: [ { step_id: 1, + source: "system", + timestamp: "2026-01-01T00:00:20Z", + message: "system prompt", + }, + { + step_id: 2, + source: "user", + timestamp: "2026-01-01T00:00:20Z", + message: "perform the task", + }, + { + step_id: 3, source: "agent", - timestamp: "2026-01-01T00:00:01Z", - message: "working", + timestamp: "2026-01-01T00:00:21Z", + message: "", tool_calls: [ { tool_call_id: "call-1", function_name: "execute_playwright_code", - arguments: { code: "return 'done'" }, + arguments: { + session_id: "session-123", + code: "await page.locator('#password').fill('secret-password'); return 'done'", + }, }, ], observation: { results: [{ source_call_id: "call-1", content: "done" }], }, + metrics: { + prompt_tokens: 100, + cached_tokens: 80, + completion_tokens: 20, + cost_usd: 0.01, + }, }, ], }); @@ -101,6 +143,20 @@ function fixture(): string { kernel_mcp_server_sha: "server-sha", clawbench_source_sha: "clawbench-sha", }); + writeJson(join(success, "steps/run/verifier/kernel-mcp-result.json"), { + expected_session_id: "session-123", + }); + writeJson( + join(success, "steps/run/verifier/data/kernel-browser-lifecycle.json"), + { + timeout_seconds: 1920, + deletion_verified: true, + events: [ + { event: "browser_created", ts: 1767225620 }, + { event: "browser_deleted", ts: 1767225640 }, + ], + }, + ); const failed = join(root, "task-two__def"); writeJson(join(failed, "result.json"), { @@ -154,6 +210,30 @@ describe("Harbor result ingestion", () => { }); }); + test("classifies ungraded step setup failures as infrastructure", () => { + const root = fixture(); + const failedPath = join(root, "task-two__def", "result.json"); + const failed = JSON.parse(readFileSync(failedPath, "utf8")) as Record< + string, + unknown + >; + delete failed.exception_info; + failed.verifier_result = null; + failed.step_results = [ + { + exception_info: { + exception_type: "RuntimeError", + exception_message: "Step setup exited with code 1", + }, + }, + ]; + writeJson(failedPath, failed); + + const arm = readBenchmarkArm({ name: "candidate", path: root }); + expect(arm.trials[1].errorClass).toBe("infra"); + expect(arm.trials[1].error).toContain("Step setup exited with code 1"); + }); + test("summarizes against the intended task denominator", () => { const summary = summarizeArm( readBenchmarkArm({ name: "candidate", path: fixture() }), @@ -207,10 +287,14 @@ describe("Harbor result ingestion", () => { expect( first.filter((event) => event.span_attributes.type === "tool"), ).toHaveLength(1); + expect( + first.filter((event) => event.span_attributes.type === "task"), + ).toHaveLength(7); const root = first.find((event) => event.span_attributes.type === "eval"); expect(root?.input).toEqual({ source: "clawbench-v2", taskName: "v2-task-one", + instruction: "perform the task", }); expect(root?.span_parents).toEqual([]); const infra = first.find( @@ -229,6 +313,55 @@ describe("Harbor result ingestion", () => { reward: 1, rewardKey: "reward_lenient", }); + const llm = first.find((event) => event.span_attributes.type === "llm"); + expect(llm?.input).toEqual([ + { + source: "system", + message: "system prompt", + toolCalls: [], + observations: [], + }, + { + source: "user", + message: "perform the task", + toolCalls: [], + observations: [], + }, + ]); + expect(llm?.output).toMatchObject({ + message: "", + toolCalls: [ + { + name: "execute_playwright_code", + arguments: { + session_id: "[REDACTED]", + code: "await page.locator('#password').fill('[REDACTED]'); return 'done'", + }, + }, + ], + }); + const tool = first.find((event) => event.span_attributes.type === "tool"); + expect(tool?.metadata).toMatchObject({ + sessionIdMatchesExpected: true, + }); + const browser = first.find( + (event) => event.span_attributes.name === "browser_session", + ); + expect(browser?.metadata).toMatchObject({ + timeoutSeconds: 1920, + deletionVerified: true, + }); + expect(llm?.metrics).toMatchObject({ + start: Date.parse("2026-01-01T00:00:20Z") / 1000, + end: Date.parse("2026-01-01T00:00:21Z") / 1000, + prompt_tokens: 100, + prompt_cached_tokens: 80, + completion_tokens: 20, + tokens: 120, + cost_usd: 0.01, + }); + expect(root?.metrics).not.toHaveProperty("input_tokens"); + expect(root?.metrics).not.toHaveProperty("cost_usd"); }); test("re-publishes the same rows and spans by deterministic ID", async () => { @@ -409,22 +542,54 @@ describe("Braintrust redaction", () => { process.env.TEST_API_KEY = "super-secret-value"; expect( redactString( - 'Bearer super-secret-value sk-proj-abcdefghijklmnop?access_token=visible&token=plain&jwt=opaque "password":"generated-password" Cookie: session=visible\nhttps://example.com/browser/live/replay-slug user@example.com', + 'Bearer super-secret-value sk-proj-abcdefghijklmnop?access_token=visible&token=plain&jwt=opaque "password":"generated-password" "session_id":"session-123" Cookie: session=visible\nhttps://example.com/browser/live/replay-slug user@example.com await page.locator("#password").fill("typed-password"); await page.fill("#password", "two-arg-secret")', ), ).toBe( - 'Bearer [REDACTED] [REDACTED]?access_token=[REDACTED]&token=[REDACTED]&jwt=[REDACTED] "password":"[REDACTED]" Cookie: [REDACTED]\nhttps://example.com/browser/live/[REDACTED] [REDACTED_EMAIL]', + 'Bearer [REDACTED] [REDACTED]?access_token=[REDACTED]&token=[REDACTED]&jwt=[REDACTED] "password":"[REDACTED]" "session_id":"[REDACTED]" Cookie: [REDACTED]\nhttps://example.com/browser/live/[REDACTED] [REDACTED_EMAIL] await page.locator("#password").fill("[REDACTED]"); await page.fill("#password", "[REDACTED]")', ); - expect( - redactValue({ - api_key: "visible", - Cookie: "session=visible", - nested: ["bt-abcdefghijklmnop"], - }), - ).toEqual({ + const redacted = redactValue({ + api_key: "visible", + Cookie: "session=visible", + session_id: "session-123", + max_output_tokens: 1000, + nested: ["bt-abcdefghijklmnop"], + }); + expect(redacted).toEqual({ api_key: "[REDACTED]", Cookie: "[REDACTED]", + session_id: "[REDACTED]", + max_output_tokens: 1000, nested: ["[REDACTED]"], }); + expect(() => assertSafeToPublish(redacted)).not.toThrow(); + expect(() => + assertSafeToPublish({ code: "page.fill('still-visible')" }), + ).toThrow("typed form value"); + expect(() => + assertSafeToPublish({ + code: "page.fill('#password', 'still-visible')", + }), + ).toThrow("typed form value"); + expect( + privateInfoRead("exec_command", { + cmd: "cat /my-info/email_credentials.json", + }), + ).toBe(true); + expect( + privateInfoRead("Bash", { + command: "cat ./my-info/alex_green_personal_info.json", + }), + ).toBe(true); + expect( + privateInfoRead("Read", { + file_path: "/workspace/my-info/email_credentials.json", + }), + ).toBe(true); + expect( + privateInfoRead("exec_command", { + cmd: "cat /my-info/kernel_browser.json", + }), + ).toBe(false); delete process.env.TEST_API_KEY; }); }); @@ -473,6 +638,7 @@ describe("benchmark workflow hardening", () => { "harbor_hypeman_version=${HARBOR_HYPEMAN_VERSION:-0.1.2}", ); expect(runner).toContain('--max-retries "${HARBOR_MAX_RETRIES:-5}"'); + expect(runner).toContain('bun "$benchmark_dir/verify-purelymail.ts"'); for (const exception of [ "APITimeoutError", "APIConnectionError", @@ -480,6 +646,8 @@ describe("benchmark workflow hardening", () => { "InternalServerError", "ConnectionRefusedError", "ExecProtocolError", + "AgentSetupTimeoutError", + "RuntimeError", ]) { expect(runner).toContain(`--retry-include ${exception}`); } diff --git a/benchmarks/harbor/results.ts b/benchmarks/harbor/results.ts index 41def57..c2024a0 100644 --- a/benchmarks/harbor/results.ts +++ b/benchmarks/harbor/results.ts @@ -20,6 +20,21 @@ export interface BenchmarkMetrics { end?: number; } +export interface BenchmarkPhase { + startedAt?: string; + finishedAt?: string; + start?: number; + end?: number; + durationMs?: number; +} + +export interface BenchmarkBrowserLifecycle { + timeoutSeconds?: number; + start?: number; + end?: number; + deletionVerified?: boolean; +} + export interface BenchmarkTrial { arm: string; id: string; @@ -35,6 +50,14 @@ export interface BenchmarkTrial { error?: string; errorClass?: "infra"; metrics: BenchmarkMetrics; + phases: { + environmentSetup?: BenchmarkPhase; + agentSetup?: BenchmarkPhase; + agentExecution?: BenchmarkPhase; + verifier?: BenchmarkPhase; + }; + browser?: BenchmarkBrowserLifecycle; + expectedBrowserSessionId?: string; kernelMcpSha?: string; clawbenchSha?: string; trajectoryPath?: string; @@ -154,6 +177,20 @@ function durationMs( return Number.isFinite(duration) && duration >= 0 ? duration : undefined; } +function phase(value: unknown): BenchmarkPhase | undefined { + const timing = object(value); + const startedAt = string(timing.started_at); + const finishedAt = string(timing.finished_at); + if (!startedAt && !finishedAt) return undefined; + return { + startedAt, + finishedAt, + start: isoSeconds(startedAt), + end: isoSeconds(finishedAt), + durationMs: durationMs(startedAt, finishedAt), + }; +} + function sumStepMetric( stepResults: unknown[], key: string, @@ -224,21 +261,40 @@ function parseTrial(arm: string, trialDir: string): BenchmarkTrial { const agentInfo = object(result.agent_info); const modelInfo = object(agentInfo.model_info); const steps = array(result.step_results); - const exception = result.exception_info; + const rewards = trialRewards(result, trialDir); const exceptionFile = join(trialDir, "exception.txt"); + const stepException = + Object.keys(rewards).length === 0 + ? steps.map((step) => object(step).exception_info).find(Boolean) + : undefined; const error = errorText( - exception ?? + result.exception_info ?? + stepException ?? (existsSync(exceptionFile) ? readFileSync(exceptionFile, "utf8") : undefined), ); - const rewards = trialRewards(result, trialDir); const startedAt = string(result.started_at); const finishedAt = string(result.finished_at); const trajectoryPath = join(trialDir, "steps/run/agent/trajectory.json"); const runManifest = readJsonIfPresent( join(trialDir, "steps/run/verifier/kernel-mcp/run-manifest.json"), ); + const kernelMcpResult = readJsonIfPresent( + join(trialDir, "steps/run/verifier/kernel-mcp-result.json"), + ); + const browserLifecycle = readJsonIfPresent( + join(trialDir, "steps/run/verifier/data/kernel-browser-lifecycle.json"), + ); + const browserEvents = array(browserLifecycle.events).map(object); + const browserCreated = browserEvents.find( + (event) => event.event === "browser_created", + ); + const browserDeleted = browserEvents.find( + (event) => event.event === "browser_deleted", + ); + const browserStart = number(browserCreated?.ts); + const browserEnd = number(browserDeleted?.ts); return { arm, @@ -274,6 +330,25 @@ function parseTrial(arm: string, trialDir: string): BenchmarkTrial { end: isoSeconds(finishedAt), ...trajectoryMetrics(trajectoryPath), }, + phases: { + environmentSetup: phase(result.environment_setup), + agentSetup: phase(result.agent_setup), + agentExecution: phase(object(steps.at(-1)).agent_execution), + verifier: phase(object(steps.at(-1)).verifier), + }, + browser: + browserStart === undefined && browserEnd === undefined + ? undefined + : { + timeoutSeconds: number(browserLifecycle.timeout_seconds), + start: browserStart, + end: browserEnd, + deletionVerified: + typeof browserLifecycle.deletion_verified === "boolean" + ? browserLifecycle.deletion_verified + : undefined, + }, + expectedBrowserSessionId: string(kernelMcpResult.expected_session_id), kernelMcpSha: string(runManifest.kernel_mcp_server_sha), clawbenchSha: string(runManifest.clawbench_source_sha), trajectoryPath: existsSync(trajectoryPath) ? trajectoryPath : undefined, diff --git a/benchmarks/harbor/verify-purelymail.test.ts b/benchmarks/harbor/verify-purelymail.test.ts new file mode 100644 index 0000000..d49bcd6 --- /dev/null +++ b/benchmarks/harbor/verify-purelymail.test.ts @@ -0,0 +1,46 @@ +import { describe, expect, test } from "bun:test"; +import { verifyPurelyMail } from "./verify-purelymail"; + +describe("PurelyMail benchmark preflight", () => { + test("creates and deletes a disposable account", async () => { + const requests: Array<{ endpoint: string; body: Record }> = + []; + const fetcher = (async (request, init) => { + requests.push({ + endpoint: new URL(String(request)).pathname.split("/").at(-1) ?? "", + body: JSON.parse(String(init?.body)), + }); + return Response.json({ type: "success" }); + }) as typeof fetch; + + await verifyPurelyMail("api-key", "example.test", fetcher); + + expect(requests.map((request) => request.endpoint)).toEqual([ + "createUser", + "deleteUser", + ]); + expect(requests[0].body).toMatchObject({ + domainName: "example.test", + enablePasswordReset: false, + sendWelcomeEmail: false, + }); + expect(requests[1].body.userName).toBe( + `${requests[0].body.userName}@example.test`, + ); + }); + + test("fails closed on API errors without printing credentials", async () => { + const fetcher = (async () => + Response.json({ + type: "error", + code: "invalidToken", + message: "Token not valid.", + })) as unknown as typeof fetch; + + await expect( + verifyPurelyMail("secret-api-key", "example.test", fetcher), + ).rejects.toThrow( + "PurelyMail createUser failed (invalidToken): Token not valid.", + ); + }); +}); diff --git a/benchmarks/harbor/verify-purelymail.ts b/benchmarks/harbor/verify-purelymail.ts new file mode 100644 index 0000000..8c8204f --- /dev/null +++ b/benchmarks/harbor/verify-purelymail.ts @@ -0,0 +1,80 @@ +#!/usr/bin/env bun +import { randomBytes, randomUUID } from "node:crypto"; + +interface PurelyMailResponse { + type?: string; + code?: string; + message?: string; +} + +function env(name: string): string { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`${name} is required`); + return value; +} + +async function request( + fetcher: typeof fetch, + apiKey: string, + endpoint: string, + body: Record, +): Promise { + const response = await fetcher(`https://purelymail.com/api/v0/${endpoint}`, { + method: "POST", + headers: { + "Purelymail-Api-Token": apiKey, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }); + if (!response.ok) { + throw new Error(`PurelyMail ${endpoint} returned HTTP ${response.status}`); + } + const result = (await response.json()) as PurelyMailResponse; + if (!result || typeof result !== "object") { + throw new Error(`PurelyMail ${endpoint} returned an invalid response`); + } + if (result.type === "error") { + const code = result.code ? ` (${result.code})` : ""; + const message = result.message ? `: ${result.message}` : ""; + throw new Error(`PurelyMail ${endpoint} failed${code}${message}`); + } + return result; +} + +export async function verifyPurelyMail( + apiKey: string, + domain: string, + fetcher: typeof fetch = fetch, +): Promise { + const local = `cbpreflight${randomUUID().replace(/-/g, "").slice(0, 12)}`; + const email = `${local}@${domain}`; + const password = randomBytes(18).toString("base64url"); + let created = false; + try { + await request(fetcher, apiKey, "createUser", { + userName: local, + domainName: domain, + password, + enablePasswordReset: false, + sendWelcomeEmail: false, + }); + created = true; + } finally { + if (created) { + await request(fetcher, apiKey, "deleteUser", { userName: email }); + } + } +} + +async function main(): Promise { + await verifyPurelyMail(env("PURELY_MAIL_API_KEY"), env("PURELY_MAIL_DOMAIN")); + process.stdout.write("PurelyMail preflight passed\n"); +} + +if (import.meta.main) { + main().catch((error) => { + console.error(error instanceof Error ? error.message : error); + process.exitCode = 1; + }); +}