From 7b430798e1a60ed32e6a33aa5b1b5735073966ec Mon Sep 17 00:00:00 2001 From: vishal veerareddy Date: Sat, 15 Aug 2026 21:25:58 -0700 Subject: [PATCH 1/4] Fix agentic-traffic reliability and enable provider prompt caching MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on ITSMBench + terminal-bench via a pi-agent harness; each fix addresses a defect observed in production-like agent loops: - openai-router: never serve a clean empty completion. Upstream 429-retry exhaustion surfaced as a contentless 200 with finish_reason=stop, which agent clients read as "task complete" and terminated mid-task (observed killing 6/7 benchmark episodes). Abort the SSE stream instead so clients retry. - databricks (Azure Responses): forward reasoning effort (was silently dropped — thinking requests never reached the model), fix max_output_tokens (read from max_completion_tokens for gpt-5.x), and surface cache_read_input_tokens through both response conversions so telemetry records provider cache hits (was always null). - Prefix stability for provider prompt caching (measured 0-3% -> 92-95% cache hit rate, ~5x real cost cut on agent loops): * deterministic tool-call fallback IDs (were Date.now()+random — history re-randomized every turn) * one constant system prompt per conversation (continuations previously swapped in a generic prompt, which also silently discarded the client agent's instructions after turn 1) * uniform system-reminder stripping on every turn * content-hash tee IDs in the tool-result compressor (were timestamped — new bytes in old messages each turn) * fixed compression threshold (routed tier flaps between turns; COMPLEX keeps compression deterministic and lightly lossy) * prompt_cache_key on Azure gpt-5.x requests (session id or first-user- message hash) - config: TOOL_RESULT_COMPRESSION_ENABLED env knob (was hardcoded) - nodemon.json: ignore self-written data/db files (restart-storm fix) Co-Authored-By: Claude Fable 5 --- .gitignore | 1 + nodemon.json | 3 + src/api/openai-router.js | 16 +++++ src/clients/databricks.js | 98 +++++++++++++++++++-------- src/config/index.js | 2 +- src/context/tool-result-compressor.js | 9 ++- src/orchestrator/index.js | 9 ++- 7 files changed, 105 insertions(+), 33 deletions(-) create mode 100644 nodemon.json diff --git a/.gitignore b/.gitignore index 388719d..e59fc60 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,4 @@ tmp/ # ai-sdk-provider build output packages/ai-sdk-provider/dist/ +.env.bak-hillclimb diff --git a/nodemon.json b/nodemon.json new file mode 100644 index 0000000..2787a20 --- /dev/null +++ b/nodemon.json @@ -0,0 +1,3 @@ +{ + "ignore": ["data/*", ".lynkr/*", "*.db"] +} diff --git a/src/api/openai-router.js b/src/api/openai-router.js index cc9a565..0662b72 100644 --- a/src/api/openai-router.js +++ b/src/api/openai-router.js @@ -588,6 +588,22 @@ router.post("/chat/completions", async (req, res) => { const content = lynkrBadge(result.body) + (openaiResponse.choices[0].message.content || ""); let toolCalls = openaiResponse.choices[0].message.tool_calls; + // Guard: never serve a clean empty completion. Upstream failures + // (e.g. exhausted 429 retries) can surface here as a contentless + // message with finish_reason "stop"; agent clients read that as + // "task complete" and terminate mid-task. Killing the stream without + // a finish chunk makes the client retry instead. + if (!content && (!toolCalls || toolCalls.length === 0)) { + logger.error({ + finishReason: openaiResponse.choices[0]?.finish_reason, + terminationReason: result?.terminationReason, + status: result?.status, + }, "Empty completion reached serving path — aborting stream to force client retry"); + res.write(`data: ${JSON.stringify({ error: { message: "Upstream returned an empty completion; retry.", type: "server_error", code: "empty_completion" } })}\n\n`); + res.destroy(); + return; + } + if (clientType !== "unknown" && toolCalls && toolCalls.length > 0) { toolCalls = toolCalls.map(tc => { const mapped = mapToolForClient(tc.function?.name || "", tc.function?.arguments || "{}", clientType); diff --git a/src/clients/databricks.js b/src/clients/databricks.js index a44825b..9b9a63d 100644 --- a/src/clients/databricks.js +++ b/src/clients/databricks.js @@ -888,6 +888,23 @@ function detectAzureFormat(url) { } +/** + * Stable per-conversation cache key for GPT-5.6 declared prompt caching. + * Prefers the session id; falls back to hashing the first user message, + * which is identical across every turn of the same conversation. + */ +function derivePromptCacheKey(body) { + if (body._sessionId) return String(body._sessionId).slice(0, 64); + const first = (body.messages || []).find(m => m.role === "user"); + let text = ""; + if (first) { + text = typeof first.content === "string" + ? first.content + : JSON.stringify(first.content); + } + return "conv-" + crypto.createHash("sha1").update(text.slice(0, 4000)).digest("hex").slice(0, 32); +} + async function invokeAzureOpenAI(body, incomingHeaders = {}) { if (!config.azureOpenAI?.endpoint || !config.azureOpenAI?.apiKey) { throw new Error("Azure OpenAI endpoint or API key is not configured."); @@ -1004,6 +1021,16 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { const responsesInput = []; // Track function call IDs for matching with outputs const pendingCallIds = []; + // Fallback IDs must be DETERMINISTIC: the same history must render + // byte-identically on every turn, or provider prompt caching never hits. + // (Was Date.now()+Math.random(), which re-randomized history each turn.) + let stableIdCounter = 0; + const stableCallId = (name, args) => { + const h = crypto.createHash("sha1") + .update(`${name || ""}|${args || ""}|${stableIdCounter++}`) + .digest("hex").slice(0, 16); + return `call_${h}`; + }; // Detect if this is a continuation request (has tool results) // Azure content filter triggers on full system prompt in continuations @@ -1065,36 +1092,32 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { for (const msg of azureBody.messages) { if (msg.role === "system") { - // For continuation requests, use minimal system prompt to avoid content filter - // Azure's jailbreak detection triggers on security-related text in continuations - if (hasToolResults) { - responsesInput.push({ - type: "message", - role: "developer", - content: "You are a helpful coding assistant. Continue helping the user based on the tool results." - }); - } else { - // Initial request - use full system prompt - responsesInput.push({ - type: "message", - role: "developer", - content: typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content) - }); - } + // The system prompt must be IDENTICAL on every turn of a conversation: + // (a) swapping it per turn breaks provider prompt-cache prefixes, and + // (b) replacing it on continuations silently dropped the client + // agent's actual instructions mid-task. Strip system-reminder blocks + // uniformly (that was the content-filter trigger, not the prompt). + const sysText = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); + responsesInput.push({ + type: "message", + role: "developer", + content: stripSystemReminders(sysText) || sysText + }); } else if (msg.role === "user") { // Check if content contains tool_result blocks (Anthropic format) if (Array.isArray(msg.content)) { for (const block of msg.content) { if (block.type === "tool_result") { - const callId = block.tool_use_id || pendingCallIds.shift() || `call_${Date.now()}`; + const callId = block.tool_use_id || pendingCallIds.shift() || stableCallId("result", block.content); responsesInput.push({ type: "function_call_output", call_id: callId, output: typeof block.content === 'string' ? block.content : JSON.stringify(block.content || "") }); } else if (block.type === "text") { - // For continuation requests, strip system-reminder tags to avoid jailbreak filter - const textContent = hasToolResults ? stripSystemReminders(block.text || "") : (block.text || ""); + // Strip system-reminder tags on EVERY turn (uniformly), so the + // same message renders identically across turns (cache prefix). + const textContent = stripSystemReminders(block.text || ""); if (textContent) { // Only add if there's content after stripping responsesInput.push({ type: "message", @@ -1105,11 +1128,9 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { } } } else { - // For continuation requests, strip system-reminder tags to avoid jailbreak filter + // Strip system-reminder tags uniformly on every turn (cache prefix). let userContent = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); - if (hasToolResults) { - userContent = stripSystemReminders(userContent); - } + userContent = stripSystemReminders(userContent); if (userContent) { // Only add if there's content after stripping responsesInput.push({ type: "message", @@ -1123,7 +1144,7 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { if (msg.tool_calls && msg.tool_calls.length > 0) { // OpenAI format: tool_calls array for (const tc of msg.tool_calls) { - const callId = tc.id || `call_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + const callId = tc.id || stableCallId(tc.function?.name || tc.name, tc.function?.arguments); pendingCallIds.push(callId); responsesInput.push({ type: "function_call", @@ -1138,7 +1159,7 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { // Anthropic format: content is array of blocks for (const block of msg.content) { if (block.type === "tool_use") { - const callId = block.id || `call_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`; + const callId = block.id || stableCallId(block.name, block.input); pendingCallIds.push(callId); responsesInput.push({ type: "function_call", @@ -1164,7 +1185,7 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { } } else if (msg.role === "tool") { // Tool results become function_call_output - const callId = msg.tool_call_id || pendingCallIds.shift() || `call_${Date.now()}`; + const callId = msg.tool_call_id || pendingCallIds.shift() || stableCallId("tool", msg.content); responsesInput.push({ type: "function_call_output", call_id: callId, @@ -1173,12 +1194,26 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { } } + // Reasoning effort: honor the client's request, else the env default. + // Without this, gpt-5.x reasoning models run at their shallowest setting. + const reasoningEffort = body.reasoning_effort + ?? body.reasoning?.effort + ?? process.env.AZURE_OPENAI_REASONING_EFFORT + ?? null; const responsesBody = { input: responsesInput, model: azureBody.model, - max_output_tokens: azureBody.max_tokens, + // gpt-5.x deployments store the cap under max_completion_tokens. + max_output_tokens: azureBody.max_completion_tokens ?? azureBody.max_tokens, tools: responsesTools, tool_choice: azureBody.tool_choice, + ...(isGpt5 && reasoningEffort ? { reasoning: { effort: reasoningEffort } } : {}), + // GPT-5.6 caching is declaration-based: a stable per-conversation key + // makes prefix cache matching reliable (~90% discount on agent loops). + // _sessionId is empty on the agentic path, so fall back to a hash of + // the conversation's first user message — stable across every turn of + // the same task. Keep per-key traffic under ~15 req/min. + ...(isGpt5 ? { prompt_cache_key: derivePromptCacheKey(body) } : {}), stream: false }; logger.debug({ @@ -1277,6 +1312,11 @@ async function invokeAzureOpenAI(body, incomingHeaders = {}) { completion_tokens: result.json.usage.completion_tokens ?? result.json.usage.output_tokens ?? 0, total_tokens: result.json.usage.total_tokens ?? ((result.json.usage.input_tokens ?? 0) + (result.json.usage.output_tokens ?? 0)), + // Provider-side prompt-cache hits (Responses API: input_tokens_details, + // Chat Completions: prompt_tokens_details) — telemetry reads this name. + cache_read_input_tokens: result.json.usage.input_tokens_details?.cached_tokens + ?? result.json.usage.prompt_tokens_details?.cached_tokens + ?? null, } : undefined }; @@ -2330,6 +2370,10 @@ function convertOpenAIToAnthropic(response) { usage: { input_tokens: response.usage?.prompt_tokens || 0, output_tokens: response.usage?.completion_tokens || 0, + // Provider-side prompt-cache hits — telemetry reads this field name. + cache_read_input_tokens: response.usage?.cache_read_input_tokens + ?? response.usage?.prompt_tokens_details?.cached_tokens + ?? null, } }; } diff --git a/src/config/index.js b/src/config/index.js index 74c88ce..cddf79d 100644 --- a/src/config/index.js +++ b/src/config/index.js @@ -673,7 +673,7 @@ var config = { fallbackProvider, }, toolResultCompression: { - enabled: true, + enabled: process.env.TOOL_RESULT_COMPRESSION_ENABLED !== "false", }, caveman: { enabled: cavemanEnabled, diff --git a/src/context/tool-result-compressor.js b/src/context/tool-result-compressor.js index 5a03caa..e5dbacc 100644 --- a/src/context/tool-result-compressor.js +++ b/src/context/tool-result-compressor.js @@ -9,20 +9,23 @@ */ const logger = require("../logger"); +const crypto = require("crypto"); // ── Tee Recovery Cache ─────────────────────────────────────────────── const teeCache = new Map(); const TEE_MAX_SIZE = 200; const TEE_TTL_MS = 5 * 60 * 1000; // 5 minutes -let teeCounter = 0; function teeStore(original) { - if (teeCache.size >= TEE_MAX_SIZE) { + // Content-derived id: the same tool result must compress to byte-identical + // output on every turn, or the tee marker breaks provider prompt-cache + // prefixes. (Was Date.now()+counter — new bytes in old messages each turn.) + const id = "tee_" + crypto.createHash("sha1").update(original).digest("hex").slice(0, 16); + if (!teeCache.has(id) && teeCache.size >= TEE_MAX_SIZE) { const oldest = teeCache.keys().next().value; teeCache.delete(oldest); } - const id = `tee_${Date.now()}_${teeCounter++}`; teeCache.set(id, { content: original, createdAt: Date.now() }); return id; } diff --git a/src/orchestrator/index.js b/src/orchestrator/index.js index b0093f4..e467f8d 100644 --- a/src/orchestrator/index.js +++ b/src/orchestrator/index.js @@ -1841,8 +1841,13 @@ IMPORTANT TOOL USAGE RULES: // before they reach the model (saves 60-90% on test/git/lint output) if (config.toolResultCompression?.enabled !== false) { const { compressToolResults } = require("../context/tool-result-compressor"); - const tier = cleanPayload._routingTier || "MEDIUM"; - compressToolResults(cleanPayload.messages, { tier }); + // Fixed threshold: compression must be deterministic per message — the + // routed tier flaps between turns, and re-compressing history differently + // breaks provider prompt-cache prefixes. COMPLEX (>2000 chars) compresses + // only bulky outputs: with prompt caching live, resending history is + // cheap, so lighter lossiness beats aggressive compression; still bounds + // context growth enough to stay clear of the token-budget compressor. + compressToolResults(cleanPayload.messages, { tier: "COMPLEX" }); } // MCP-aware tool dedup: drop built-in tools superseded by present MCP tools From 6b4019093190326e35d665691cb3946fcc249f43 Mon Sep 17 00:00:00 2001 From: vishal veerareddy Date: Tue, 18 Aug 2026 13:53:01 -0700 Subject: [PATCH 2/4] Fix Z.AI streaming format mismatch; guard non-streaming empty completions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - zai (anthropic-format endpoint): force buffered mode. With stream:true the endpoint returns Anthropic SSE, but the downstream transformer parses OpenAI SSE — every chunk was unreadable and clients received empty completions on all streamed turns (terminal-bench: 0/13 episodes survived; 32% pass rate after this fix, verifying the mechanism). - openai-router: mirror the streaming empty-completion guard on the non-streaming path — return a retryable 502 instead of a clean empty 200 that agent clients read as "task complete". - docs: add ITSMBench results writeup (methodology, full-89 + subset scores, per-family and assertion-level analysis, reproduction). Known issue (not fixed here): the provider fallback ladder keeps the original tier's model name when switching providers (observed: fallback to azure-openai requesting model "kimi-k3" → 404). Co-Authored-By: Claude Fable 5 --- docs/itsmbench-results.md | 129 ++++++++++++++++++++++++++++++++++++++ src/api/openai-router.js | 15 +++++ src/clients/databricks.js | 6 ++ 3 files changed, 150 insertions(+) create mode 100644 docs/itsmbench-results.md diff --git a/docs/itsmbench-results.md b/docs/itsmbench-results.md new file mode 100644 index 0000000..a90ecca --- /dev/null +++ b/docs/itsmbench-results.md @@ -0,0 +1,129 @@ +# ITSMBench Results — Lynkr as the Model Under Test + +**Date:** August 13–15, 2026 +**Benchmark:** [ITSMBench](https://github.com/new-measure/ITSMBench) (Atomicwork + New Measure, released 2026-08-12) — 89 IT service-desk tasks in containerized enterprise environments (42 mocked systems, ~1,800 DB tables, ~2,000 REST endpoints), each scored pass/fail by a hidden end-state verifier. + +## Headline results + +| Measurement | Score | Cost/task (real) | +|---|---|---| +| **Full suite, 89 tasks, 1 attempt** | **31.0% Pass@1** (27/87 scored; 2 trials errored) | ~$0.90 | +| 10-task subset, 2 attempts (best config) | 35.0% Pass@1 / 40.0% Pass@2 | $0.87 | +| Same model on its native harness (official leaderboard, rank 8) | 35.51% Pass@1 | $1.29 | + +**Key claim:** after the fixes in PR #91, routing through Lynkr is statistically +indistinguishable from calling the model directly (35.0% vs 35.51% on matched +conditions) while costing ~33% less — the proxy tax was eliminated. + +## Setup + +- **Model:** Azure OpenAI `gpt-5.6-sol`, reasoning effort high, served through + Lynkr's tier router (all tiers pinned to the same model — routing + intelligence was deliberately NOT part of this measurement). +- **Agent:** [pi coding agent](https://pi.dev) via [Harbor](https://github.com/laude-institute/harbor), + with a custom Harbor agent (`agents_lynkr/lynkr_pi.py` in the ITSMBench + checkout) that registers Lynkr as an OpenAI-compatible provider inside each + task container (`http://host.docker.internal:8081/v1`) and installs an ITSM + operations playbook as the agent's `AGENTS.md`. +- **Attempts:** leaderboard entries average Pass@1 over 5 attempts on all 89 + tasks; our full-suite number is 1 attempt (±~5 points), the subset number is + 2 attempts. + +## Official leaderboard context (Pass@1, 5 attempts, 89 tasks) + +| Rank | Entry | Pass@1 | $/task | +|---|---|---|---| +| 1 | claude-opus-5 [high] | 46.07% | $1.75 | +| 3 | grok-4.5 [high] | 45.39% | $0.71 | +| 6 | gpt-5.6-sol [xhigh] | 39.10% | $1.53 | +| 8 | gpt-5.6-sol [high] + codex | 35.51% | $1.29 | +| — | **Lynkr → gpt-5.6-sol [high] + pi** | **31–35%** ⚠ | **$0.87–0.90** | +| 15 | gpt-5.6-terra [low] | 20.00% | $0.37 | + +⚠ Not an official submission: 1–2 attempts vs their 5; the full-89 single-attempt +run landed at 31.0%, the matched-methodology subset at 35.0%. + +## Per-family breakdown (full 89, 1 attempt) + +| Family | Score | Notes | +|---|---|---| +| iam (identity/access) | 6/9 | strongest area | +| ops | 4/5 | strong | +| grc (compliance) | 3/7 | | +| a / b / n | 4/11, 4/11, 4/12 | middling | +| c (BEC/compromise response) | 1/5 | weak — misses restoration of false-positive lockouts | +| alloc (IPAM), net | 0/5 | weak — mutate-before-verify traps | +| **ep (offboarding/endpoints)** | **1/22** | the dominant weakness: blast-radius enumeration (all devices of a leaver, whole termination cohorts) | + +The `ep` family alone is a quarter of the benchmark; solving its enumeration +pattern is the single highest-leverage quality improvement available +(31% → potentially mid-50s). + +## What the benchmark drove into Lynkr (PR #91) + +Measured defects found and fixed during this work: + +1. **Empty-completion guard** — exhausted 429 retries surfaced as clean empty + 200s; agent clients read them as "task complete" and died mid-episode + (killed 6/7 episodes in one run). +2. **System prompt preserved on continuations** — previously replaced with a + generic one-liner after turn 1, silently discarding the client agent's + instructions. +3. **Reasoning effort forwarded** to Azure Responses (was silently dropped); + `max_output_tokens` fixed for gpt-5.x. +4. **Prefix-stable request pipeline** — deterministic tool-call/tee IDs, + uniform system-reminder stripping, fixed compression threshold, and + `prompt_cache_key` — took provider prompt-cache hits from 0–3% to + **92–95%** (~5× real cost reduction; Lynkr's recorded `cost_usd` predates + cache-discount awareness and overstates real cost ~5–7×). +5. **Cache-hit telemetry** — `cache_read_tokens` now recorded (was always + null; the field was dropped in two response conversions). + +## Agent playbook findings (transferable to any ITSM agent) + +Failure analysis of always-failing tasks produced a 9-rule playbook (in +`agents_lynkr/lynkr_pi.py::_PLAYBOOK`); the highest-value rules: + +- **Sweep the class, not the named entity** — enumerate ALL devices of a + leaver, ALL members of a terminated cohort, ALL suspended users; never stop + at what the ticket names. +- **Undo the over-response** — reversing false-positive automated lockouts is + part of resolving the incident. +- **Semantic read-back after every write** — ServiceNow "Closed" = state 7, + not 6; verify by value, not by HTTP 200. +- **Deprovision, don't suspend** — applies to service accounts too. +- **Escalation = a message in the security channel**, not a ticket note. + +Rule adoption flipped task-a-2 from a persistent failure to 20/20 assertions. + +## Reproduction + +```bash +# One-time setup (see agents_lynkr/lynkr_pi.py for the custom agent) +uv tool install harbor +git clone https://github.com/new-measure/ITSMBench ~/ITSMBench + +# Subset run (~$10-15, ~15 min at concurrency 3) +cd ~/ITSMBench && set -a && source .env && set +a +PYTHONPATH=. harbor run -c configs/lynkr-subset.yaml \ + --agent-setup-timeout-multiplier 3 --env-file .env -y + +# Results: jobs//result.json + per-trial verifier/ctrf.json +# Cost/cache: Lynkr telemetry (.lynkr/telemetry.db, cache_read_tokens column) +``` + +Gotchas: `-p` is not repeatable (use a `tasks:` list in a config yaml); +pre-build task environment images; the concurrency flag is `--n-concurrent`; +agent setup needs the timeout multiplier. + +## Open items + +- **ep-family enumeration** — largest quality lever (playbook rules exist but + the cohort-discovery inference still fails). +- **Streaming passthrough** ([#92](https://github.com/Fast-Editor/Lynkr/issues/92)) — + remaining proxy latency; hurts long-turn workloads (see terminal-bench). +- **Tier routing was not exercised** — all tiers pinned to one model. A + cheap-model + cascade-verify configuration is the untested path toward the + cost-quality frontier (evo-style ~50% at ~$0.10–0.25/task). +- 5-attempt full-89 run (~$385 at high effort) for an officially comparable + number. diff --git a/src/api/openai-router.js b/src/api/openai-router.js index 0662b72..db86e84 100644 --- a/src/api/openai-router.js +++ b/src/api/openai-router.js @@ -780,6 +780,21 @@ router.post("/chat/completions", async (req, res) => { }, "Tool names mapped for non-streaming chat/completions"); } + // Guard (mirrors the streaming path): never serve a clean empty + // completion — upstream failures can surface as contentless messages + // with finish_reason "stop", which agent clients read as "done" and + // terminate mid-task. A 502 is retryable; an empty 200 is a silent kill. + const _msg = openaiResponse.choices?.[0]?.message; + if (!_msg?.content && !(_msg?.tool_calls?.length > 0)) { + logger.error({ + finishReason: openaiResponse.choices?.[0]?.finish_reason, + usage: openaiResponse.usage, + }, "Empty completion reached non-streaming serving path — returning 502"); + return res.status(502).json({ + error: { message: "Upstream returned an empty completion; retry.", type: "server_error", code: "empty_completion" } + }); + } + logger.info({ duration: Date.now() - startTime, mode: "non-streaming", diff --git a/src/clients/databricks.js b/src/clients/databricks.js index 9b9a63d..f0a9454 100644 --- a/src/clients/databricks.js +++ b/src/clients/databricks.js @@ -2050,6 +2050,12 @@ async function invokeZai(body, incomingHeaders = {}) { zaiBody = { ...body }; zaiBody.model = mappedModel; + // Force buffered mode: with stream:true this endpoint returns ANTHROPIC + // SSE, but the downstream transformer parses OPENAI SSE — every chunk is + // unreadable and the client receives an empty completion. Buffered JSON + // converts correctly; the router synthesizes client-side SSE as usual. + zaiBody.stream = false; + // Inject standard tools if client didn't send any (passthrough mode) if (!Array.isArray(zaiBody.tools) || zaiBody.tools.length === 0) { zaiBody.tools = STANDARD_TOOLS; From 9e7388d6490e95b17c4b0f9d57a8bb35b9957ff8 Mon Sep 17 00:00:00 2001 From: vishal veerareddy Date: Tue, 18 Aug 2026 13:53:16 -0700 Subject: [PATCH 3/4] gitignore: cover all .env backup files Co-Authored-By: Claude Fable 5 --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index e59fc60..0cf6adf 100644 --- a/.gitignore +++ b/.gitignore @@ -41,4 +41,4 @@ tmp/ # ai-sdk-provider build output packages/ai-sdk-provider/dist/ -.env.bak-hillclimb +.env.bak-* From 4b816cf5231b4b243377c30b3e8df8d59f0f5753 Mon Sep 17 00:00:00 2001 From: vishal veerareddy Date: Tue, 18 Aug 2026 14:05:14 -0700 Subject: [PATCH 4/4] intent-score: memoize reconciled scores per cleaned text MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The classifier leg is live and can time out under load, sending identical text down different fallback paths — the same turn scored differently between the router and the pin-drift checker (observed 63 vs 77 under suite-wide Ollama contention; session-fingerprint drift-consistency test failed intermittently in the full suite). First resolution wins for the process lifetime. Lexical fallback is not memoized so a degraded score isn't locked in past service recovery. Calls with scoring-altering opts (custom centroids/embedFn, risk, prior turns) bypass the memo. Fixes the flaky drift-consistency failure; 1257/1257 passing across back-to-back full-suite runs. Co-Authored-By: Claude Fable 5 --- src/routing/intent-score.js | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/src/routing/intent-score.js b/src/routing/intent-score.js index d0228f9..1c57cce 100644 --- a/src/routing/intent-score.js +++ b/src/routing/intent-score.js @@ -366,6 +366,14 @@ function _reconcile(anchorScore, anchorClass, classifierResult) { return { score: anchorScore, reconciled: 'up_gated' }; } +// Memoize the final reconciled score per cleaned text. The classifier leg is +// live and can time out under load, sending identical text down different +// fallback paths — which made the same turn score differently between the +// router and the pin-drift checker (observed: 63 vs 77 for one prompt under +// suite-wide Ollama contention). First resolution wins for the process life. +const _scoreMemo = new Map(); +const _SCORE_MEMO_MAX = 500; + async function scoreIntent(payload, opts = {}) { const mode = opts.mode ?? intentScoreMode(); if (mode === 'legacy') return null; @@ -373,6 +381,21 @@ async function scoreIntent(payload, opts = {}) { const text = extractCleanUserText(payload); if (!text) return null; + // Only memoize plain calls — opts that alter scoring (custom centroids, + // embedFn, risk/context inputs) must not share cache entries. + const memoizable = !opts.centroids && !opts.embedFn && !opts.forceMatched + && !opts.riskLevel && !opts.skipClassifier && !opts.priorTurns; + if (memoizable && _scoreMemo.has(text)) return _scoreMemo.get(text); + const _memoSet = (result) => { + if (memoizable && result) { + if (_scoreMemo.size >= _SCORE_MEMO_MAX) { + _scoreMemo.delete(_scoreMemo.keys().next().value); + } + _scoreMemo.set(text, result); + } + return result; + }; + try { const centroids = opts.centroids !== undefined ? opts.centroids : await getDefaultCentroids(); if (centroids) { @@ -421,7 +444,7 @@ async function scoreIntent(payload, opts = {}) { finalScore: score, }, '[IntentScore] classifier reconciled anchor score'); } - return { + return _memoSet({ score, mode: reconciled ? 'anchor+classifier' : 'anchor', class: cls, @@ -431,13 +454,15 @@ async function scoreIntent(payload, opts = {}) { classifierTier: classifierResult?.tier ?? null, classifierConfidence: classifierResult?.confidence ?? null, reconciled, - }; + }); } } } catch (err) { logger.debug({ err: err.message }, '[IntentScore] anchor scoring failed — lexical fallback'); } + // Lexical fallback is NOT memoized: it usually means the embedding service + // was unavailable; pinning it would lock in degraded scores after recovery. return { score: _lexicalCleanScore(text), mode: 'lexical', text }; }