diff --git a/docs/architecture.md b/docs/architecture.md index d909c1f..ed295c3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -85,7 +85,7 @@ The router supports: - `fallback`: members are attempted in their configured order. - `round-robin`: each request rotates the starting member, then retains fallback behavior through the remaining members. -Timeouts, `408`, `429`, `5xx`, and provider/model capability incompatibilities such as an unsupported model can advance fallback. Capability failures do not create account cooldown locks. The per-member timeout defaults to 60 seconds. An explicit route stops immediately on other non-retryable failures; a later account lock cannot replace that terminal response. +Named routes and OAuth account pools continue through every untried target until an upstream returns a usable `2xx` response. Any non-`2xx` status advances fallback regardless of error type. A `2xx` response that contains an immediate stream error, ends without usable output, has no response body, contains invalid JSON, or carries an explicit error payload also advances fallback. Failure classification controls account cooldown locks and diagnostics; it never stops routing. Capability failures do not create account cooldown locks. The per-member timeout defaults to 60 seconds. Caller cancellation stops immediately, and a stream cannot be transparently rerouted after output has already reached the client. OAuth providers add an account-pool layer beneath model routing. Accounts receive monotonic, never-reused aliases (`oauth1`, `oauth2`, ...). Model discovery advertises one canonical pooled id such as `chatgpt/gpt-5.4`; account-qualified ids such as `chatgpt/oauth2/gpt-5.4` and legacy stored-account ids remain resolvable but are not advertised. Quota failures create an account-wide lock using provider reset hints when available; authentication and transient failures use shorter model-scoped cooldowns. Early streaming quota events are inspected before the client stream starts so fallback can still occur. Selection, failure, fallback, locked-account skips, and terminal exhaustion are written as structured logs. diff --git a/package-lock.json b/package-lock.json index 4099e71..dfcd9bb 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@gitcommit90/rerouted", - "version": "0.5.0", + "version": "0.5.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@gitcommit90/rerouted", - "version": "0.5.0", + "version": "0.5.1", "license": "MIT", "bin": { "rerouted": "src/cli/index.js" diff --git a/package.json b/package.json index 5875149..85e8f9f 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@gitcommit90/rerouted", "productName": "ReRouted", - "version": "0.5.0", + "version": "0.5.1", "description": "A local AI router for connected accounts, models, named routes, and automatic fallback.", "author": "gitcommit90", "license": "MIT", diff --git a/src/lib/router.js b/src/lib/router.js index dc9e949..14cbbbd 100644 --- a/src/lib/router.js +++ b/src/lib/router.js @@ -181,7 +181,7 @@ function orderMembers(resolved, rrState) { } function isRetryableStatus(status) { - return status === 401 || status === 403 || status === 408 || status === 429 || status >= 500; + return !(status >= 200 && status < 300); } function humanProviderName(attempt) { @@ -285,6 +285,11 @@ function parseResetHint(response, bodyText, now = Date.now()) { function parseEarlyResponsesFailure(text, response) { const events = String(text || "").split(/\r?\n\r?\n/); for (const event of events) { + const eventType = event + .split(/\r?\n/) + .find((line) => line.startsWith("event:")) + ?.slice(6) + .trim(); const dataText = event .split(/\r?\n/) .filter((line) => line.startsWith("data:")) @@ -302,15 +307,31 @@ function parseEarlyResponsesFailure(text, response) { : data?.response?.error && typeof data.response.error === "object" ? data.response.error : data; + const explicitFailure = + eventType === "error" || + data?.type === "error" || + data?.type === "response.failed" || + !!data?.error || + !!data?.response?.error; + if (!explicitFailure) continue; const signature = [data?.type, error?.type, error?.code, error?.message] .filter(Boolean) .join(" "); - if (!/usage[ _-]?limit|rate[ _-]?limit|quota|resource[ _-]?exhaust|insufficient[ _-]?quota/i.test(signature)) { - continue; - } - const message = error?.message || data?.message || "Upstream usage limit reached"; + const quota = + /usage[ _-]?limit|rate[ _-]?limit|quota|resource[ _-]?exhaust|insufficient[ _-]?quota/i.test( + signature + ); + const reportedStatus = Number(error?.status ?? data?.status); + const status = + Number.isInteger(reportedStatus) && reportedStatus >= 400 && reportedStatus <= 599 + ? reportedStatus + : quota + ? 429 + : 502; + const message = + error?.message || data?.message || error?.code || error?.type || "Upstream stream failed"; return { - status: 429, + status, error: message, resetAt: parseResetHint(response, JSON.stringify(error)), }; @@ -318,6 +339,38 @@ function parseEarlyResponsesFailure(text, response) { return null; } +function nonStreamingFailure(text, response) { + const bodyText = String(text || "").trim(); + let status = 502; + let error = bodyText || "Upstream returned a non-streaming response to a streaming request"; + try { + const parsed = JSON.parse(bodyText); + const upstream = + parsed?.error && typeof parsed.error === "object" + ? parsed.error + : parsed?.response?.error && typeof parsed.response.error === "object" + ? parsed.response.error + : parsed; + const reportedStatus = Number(upstream?.status ?? parsed?.status); + if (Number.isInteger(reportedStatus) && reportedStatus >= 400 && reportedStatus <= 599) { + status = reportedStatus; + } + error = + upstream?.message || + upstream?.detail || + upstream?.code || + upstream?.type || + error; + } catch { + /* preserve the plain-text upstream response */ + } + return { + status, + error: String(error).slice(0, 500), + resetAt: parseResetHint(response, bodyText), + }; +} + function hasProductiveResponsesEvent(text) { const events = String(text || "").split(/\r?\n\r?\n/); for (const event of events) { @@ -330,9 +383,6 @@ function hasProductiveResponsesEvent(text) { try { const data = JSON.parse(dataText); const type = String(data?.type || ""); - if (type === "response.completed" || type === "response.done" || type === "message_stop") { - return true; - } if ( type === "response.output_text.delta" && ((typeof data?.delta === "string" && data.delta.length > 0) || @@ -340,6 +390,29 @@ function hasProductiveResponsesEvent(text) { ) { return true; } + if ( + type === "response.function_call_arguments.delta" && + typeof data?.delta === "string" && + data.delta.length > 0 + ) { + return true; + } + if ( + (type === "response.output_item.added" || type === "response.output_item.done") && + (data?.item?.type === "function_call" || data?.item?.type === "custom_tool_call") && + typeof data.item.name === "string" && + data.item.name.length > 0 + ) { + return true; + } + if ( + type === "content_block_start" && + data?.content_block?.type === "tool_use" && + typeof data.content_block.name === "string" && + data.content_block.name.length > 0 + ) { + return true; + } if ( type === "content_block_delta" && [data?.delta?.text, data?.delta?.partial_json].some( @@ -352,8 +425,8 @@ function hasProductiveResponsesEvent(text) { if ( choices.some( (choice) => - choice?.finish_reason != null || (typeof choice?.delta?.content === "string" && choice.delta.content.length > 0) || + (typeof choice?.delta?.refusal === "string" && choice.delta.refusal.length > 0) || (typeof choice?.delta?.reasoning_content === "string" && choice.delta.reasoning_content.length > 0) || (Array.isArray(choice?.delta?.tool_calls) && choice.delta.tool_calls.length > 0) @@ -366,9 +439,10 @@ function hasProductiveResponsesEvent(text) { Array.isArray(candidates) && candidates.some( (candidate) => - candidate?.finishReason != null || (candidate?.content?.parts || []).some( - (part) => typeof part?.text === "string" && part.text.length > 0 + (part) => + (typeof part?.text === "string" && part.text.length > 0) || + (typeof part?.functionCall?.name === "string" && part.functionCall.name.length > 0) ) ) ) { @@ -381,6 +455,26 @@ function hasProductiveResponsesEvent(text) { return false; } +function isUsableChatCompletion(payload) { + return ( + payload && + typeof payload === "object" && + Array.isArray(payload.choices) && + payload.choices.some( + (choice) => { + const message = choice?.message; + if (!message || typeof message !== "object") return false; + if (typeof message.content === "string" && message.content.length > 0) return true; + if (Array.isArray(message.content) && message.content.length > 0) return true; + if (Array.isArray(message.tool_calls) && message.tool_calls.length > 0) return true; + if (message.function_call && typeof message.function_call === "object") return true; + if (typeof message.refusal === "string" && message.refusal.length > 0) return true; + return false; + } + ) + ); +} + function parseSseError(text) { const events = String(text || "").split(/\r?\n\r?\n/); for (const event of events) { @@ -484,23 +578,32 @@ function rebuildResponseWithPrelude(response, chunks, reader) { }); } -async function inspectEarlyResponsesSse(response, maxBytes = 64 * 1024) { +async function inspectEarlyResponsesSse(response) { if (!response?.ok || !response.body?.getReader) return { response, failure: null }; const contentType = String(response.headers?.get?.("content-type") || "").toLowerCase(); - if (contentType && !contentType.includes("text/event-stream")) return { response, failure: null }; + if (contentType && !contentType.includes("text/event-stream")) { + const text = await response.text().catch(() => ""); + try { + const payload = JSON.parse(text); + if (isUsableChatCompletion(payload)) { + return { response: null, failure: null, openAiJson: payload }; + } + } catch { + /* classify the non-streaming body below */ + } + return { response: null, failure: nonStreamingFailure(text, response) }; + } const reader = response.body.getReader(); const decoder = new TextDecoder(); const chunks = []; let text = ""; - let bytes = 0; let done = false; - while (bytes < maxBytes) { + while (true) { const next = await reader.read(); done = next.done; if (done) break; chunks.push(next.value); - bytes += next.value?.byteLength || 0; text += decoder.decode(next.value, { stream: true }); const failure = parseEarlyResponsesFailure(text, response); if (failure) { @@ -513,6 +616,24 @@ async function inspectEarlyResponsesSse(response, maxBytes = 64 * 1024) { } if (done) { + try { + const payload = JSON.parse(text); + if (isUsableChatCompletion(payload)) { + return { response: null, failure: null, openAiJson: payload }; + } + } catch { + /* the completed body may be SSE */ + } + if (!hasProductiveResponsesEvent(text)) { + return { + response: null, + failure: { + status: 502, + error: "Upstream stream ended before producing a usable response", + resetAt: null, + }, + }; + } const body = new ReadableStream({ start(controller) { for (const chunk of chunks) controller.enqueue(chunk); @@ -646,7 +767,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, if ( !provider?.id || !isOAuthProvider(provider) || - !result?.fallbackEligible || + !result?.cooldownEligible || result.failureKind === "capability" ) { return null; @@ -731,13 +852,56 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, ok: false, status: inspected.failure.status, error: inspected.failure.error, - retryable: classification.eligible, - fallbackEligible: classification.eligible, + cooldownEligible: classification.eligible, failureKind: classification.kind, defaultCooldownMs: classification.defaultCooldownMs, resetAt: inspected.failure.resetAt, }; } + if (inspected.openAiJson) { + if (stream) { + bound.clearRequestTimeout(); + cleanupDeferred = true; + return { + ok: true, + status: res.status, + streamPipe: async (clientRes) => { + try { + const completion = inspected.openAiJson; + const chunk = { + id: completion.id || `chatcmpl-${Date.now()}`, + object: "chat.completion.chunk", + created: completion.created || Math.floor(Date.now() / 1000), + model: completion.model || upstreamModel, + choices: completion.choices.map((choice, index) => ({ + index: choice.index ?? index, + delta: choice.message || {}, + finish_reason: choice.finish_reason ?? "stop", + })), + }; + clientRes.write(`data: ${JSON.stringify(chunk)}\n\n`); + clientRes.write("data: [DONE]\n\n"); + return completion.usage || null; + } finally { + bound.cleanup(); + } + }, + providerId: provider.id, + providerType: provider.type, + providerName: provider.name, + model: upstreamModel, + }; + } + return { + ok: true, + status: res.status, + openAiJson: inspected.openAiJson, + providerId: provider.id, + providerType: provider.type, + providerName: provider.name, + model: upstreamModel, + }; + } res = inspected.response; } const status = res.status; @@ -750,8 +914,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, ok: false, status, error, - retryable: classification.eligible, - fallbackEligible: classification.eligible, + cooldownEligible: classification.eligible, failureKind: classification.kind, defaultCooldownMs: classification.defaultCooldownMs, resetAt: parseResetHint(res, text), @@ -759,6 +922,16 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, } if (stream) { + if (!res.body) { + return { + ok: false, + status: 502, + error: "Upstream returned no response body", + cooldownEligible: true, + failureKind: "transient", + defaultCooldownMs: COOLDOWN_MS.transient, + }; + } // Response selection succeeded; keep client cancellation, not the header timeout. bound.clearRequestTimeout(); cleanupDeferred = true; @@ -817,6 +990,16 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, collect: true, reasoningScope: meta.reasoningScope, }); + if (!isUsableChatCompletion(collected)) { + return { + ok: false, + status: 502, + error: "Upstream returned no usable completion", + cooldownEligible: true, + failureKind: "transient", + defaultCooldownMs: COOLDOWN_MS.transient, + }; + } return { ok: true, status: 200, @@ -829,12 +1012,40 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, } const raw = await res.json(); + if (raw?.error || raw?.response?.error) { + const error = errorMessageFromText(JSON.stringify(raw), "Upstream returned an error payload"); + const reportedStatus = Number(raw?.error?.status ?? raw?.response?.error?.status); + const errorStatus = + Number.isInteger(reportedStatus) && reportedStatus >= 400 && reportedStatus <= 599 + ? reportedStatus + : 502; + const classification = classifyFailure(errorStatus, error); + return { + ok: false, + status: errorStatus, + error, + cooldownEligible: classification.eligible, + failureKind: classification.kind, + defaultCooldownMs: classification.defaultCooldownMs, + resetAt: parseResetHint(res, JSON.stringify(raw)), + }; + } let openAiJson = raw; if (meta.translate === true || provider.type === "claude") { openAiJson = claude.fromAnthropicJson(raw, upstreamModel); } else if (meta.translate === "gemini" || provider.type === "antigravity") { openAiJson = antigravity.fromGeminiJson(raw, upstreamModel); } + if (!isUsableChatCompletion(openAiJson)) { + return { + ok: false, + status: 502, + error: "Upstream returned no usable completion", + cooldownEligible: true, + failureKind: "transient", + defaultCooldownMs: COOLDOWN_MS.transient, + }; + } return { ok: true, status: 200, @@ -850,8 +1061,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, ok: false, status: 499, error: "Client disconnected", - retryable: false, - fallbackEligible: false, + cooldownEligible: false, canceled: true, failureKind: "canceled", defaultCooldownMs: 0, @@ -862,8 +1072,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, ok: false, status: 408, error: e.message || "Upstream request timeout", - retryable: true, - fallbackEligible: true, + cooldownEligible: true, failureKind: "transient", defaultCooldownMs: COOLDOWN_MS.transient, }; @@ -872,8 +1081,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, ok: false, status: 502, error: e.message || String(e), - retryable: true, - fallbackEligible: true, + cooldownEligible: true, failureKind: "transient", defaultCooldownMs: COOLDOWN_MS.transient, }; @@ -915,9 +1123,9 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, let fallbackFrom = null; const attemptedOAuthPools = new Set(); let sawNonOAuthAttempt = false; - let stoppedWithoutFallback = false; + let lastAttemptedFailure = null; - routeMembers: for (const member of ordered) { + for (const member of ordered) { const accounts = member.accounts?.length ? member.accounts : [member.provider]; for (let accountIndex = 0; accountIndex < accounts.length; accountIndex += 1) { const provider = accounts[accountIndex]; @@ -1002,8 +1210,7 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, ok: false, status, error: e.message || String(e), - retryable: classification.eligible, - fallbackEligible: classification.eligible, + cooldownEligible: classification.eligible, failureKind: classification.kind, defaultCooldownMs: classification.defaultCooldownMs, }; @@ -1073,31 +1280,28 @@ function createRouter({ store, fetchImpl = fetch, requestLog, timeoutMs, usage, status: result.status, error: result.error, failureKind: result.failureKind, - retryable: !!result.fallbackEligible, + retryable: true, lockedUntil: savedLock?.until || null, }; errors.push(failed); + lastAttemptedFailure = failed; logger.warn(oauthAttempt ? "router account failure" : "router route member failure", { event: oauthAttempt ? "account_failure" : "route_member_failure", routeModel: modelId, ...failed, }); fallbackFrom = failed; - if (!result.fallbackEligible) { - stoppedWithoutFallback = true; - break routeMembers; - } } } - const last = errors[errors.length - 1] || { status: 502, error: "All providers failed" }; + const last = lastAttemptedFailure || + errors[errors.length - 1] || { status: 502, error: "All providers failed" }; const attemptsSummary = errors .map((attempt) => { return `${attemptLabel(attempt)} [${attempt.status || 502}]: ${attempt.error}`; }) .join("; "); - const accountsExhausted = - attemptedOAuthPools.size === 1 && !sawNonOAuthAttempt && !stoppedWithoutFallback; + const accountsExhausted = attemptedOAuthPools.size === 1 && !sawNonOAuthAttempt; const terminalMessage = accountsExhausted ? `All eligible accounts failed for ${modelId}. Attempts: ${attemptsSummary || "none"}` : `Route failed for ${modelId}. Attempts: ${attemptsSummary || "none"}`; diff --git a/tests/gateway.test.js b/tests/gateway.test.js index 7a2311b..be7e566 100644 --- a/tests/gateway.test.js +++ b/tests/gateway.test.js @@ -609,10 +609,12 @@ describe("gateway request limits", () => { }); describe("combo ordering + retryable", () => { - it("marks 429/5xx as retryable", () => { + it("marks every non-2xx status as retryable", () => { assert.equal(isRetryableStatus(429), true); assert.equal(isRetryableStatus(503), true); - assert.equal(isRetryableStatus(400), false); + assert.equal(isRetryableStatus(400), true); + assert.equal(isRetryableStatus(200), false); + assert.equal(isRetryableStatus(204), false); }); it("round-robin rotates starting index", () => { diff --git a/tests/router-fallback.test.js b/tests/router-fallback.test.js index d55e337..bb12952 100644 --- a/tests/router-fallback.test.js +++ b/tests/router-fallback.test.js @@ -906,7 +906,7 @@ describe("same-provider OAuth account fallback", () => { }); }); - it("does not claim account exhaustion after a non-fallback request error", async () => { + it("exhausts every OAuth account after generic request errors", async () => { const store = createStore(tmpConfig()); store.seed({ providers: [oauthAccount("prov_a", "token-a", 100), oauthAccount("prov_b", "token-b", 200)], @@ -933,9 +933,9 @@ describe("same-provider OAuth account fallback", () => { }); assert.equal(result.ok, false); - assert.deepEqual(calls, ["token-a"]); - assert.ok(logger.entries.some((entry) => entry.meta?.event === "route_failure_no_fallback")); - assert.ok(!logger.entries.some((entry) => entry.meta?.event === "accounts_exhausted")); + assert.deepEqual(calls, ["token-a", "token-b"]); + assert.equal(result.error.error.details.length, 2); + assert.ok(logger.entries.some((entry) => entry.meta?.event === "accounts_exhausted")); }); it("does not persist OAuth account locks for keyed providers", async () => { @@ -1125,21 +1125,12 @@ describe("same-provider OAuth account fallback", () => { assert.ok(logger.entries.some((entry) => entry.meta?.event === "account_fallback")); }); - it("stops an explicit route on a non-retryable failure before later locks can replace it", async () => { + it("continues an explicit route after a generic 400 until a usable 2xx", async () => { const store = createStore(tmpConfig()); store.seed({ providers: [ oauthAccount("prov_a", "token-a", 100), - oauthAccount("prov_b", "token-b", 200, { - modelLocks: { - "*": { - until: Date.now() + 5 * 60_000, - status: 429, - kind: "quota", - reason: "stale transport lock", - }, - }, - }), + oauthAccount("prov_b", "token-b", 200), ], combos: [ { @@ -1160,9 +1151,11 @@ describe("same-provider OAuth account fallback", () => { logger: captureLogger(), usage: { record: (entry) => usageRows.push(entry) }, fetchImpl: async (_url, options) => { - calls.push(authToken(options)); + const token = authToken(options); + calls.push(token); + if (token === "token-b") return successResponse("fallback worked"); return new Response(JSON.stringify({ error: { message: "not-found" } }), { - status: 404, + status: 400, headers: { "Content-Type": "application/json" }, }); }, @@ -1176,18 +1169,202 @@ describe("same-provider OAuth account fallback", () => { }, }); - assert.equal(result.ok, false); - assert.equal(result.status, 404); - assert.deepEqual(calls, ["token-a"]); - assert.equal(result.error.error.details.length, 1); - assert.match(result.error.error.message, /xAI \(Grok\) \(oauth1\) \[404\]: not-found/); - assert.doesNotMatch(result.error.error.message, /locked|prov_/i); + assert.equal(result.ok, true, JSON.stringify(result.error)); + assert.equal(result.openAiJson.choices[0].message.content, "fallback worked"); + assert.deepEqual(calls, ["token-a", "token-b"]); assert.deepEqual( usageRows.map((row) => [row.providerType, row.providerName, row.accountAlias]), - [["xai", "xAI (Grok)", "oauth1"]] + [["xai", "prov_b", "oauth2"]] ); }); + it("continues through a 429 and provider-specific 400 to the next route member", async () => { + const store = createStore(tmpConfig()); + const providers = ["Claude", "NVIDIA NIM", "Backup"].map((name, index) => ({ + id: `prov_${index + 1}`, + type: "openai-compat", + name, + baseUrl: `https://provider-${index + 1}.test/v1`, + apiKey: `key-${index + 1}`, + enabled: true, + models: [{ id: `model-${index + 1}`, name: `Model ${index + 1}`, enabled: true }], + })); + store.seed({ + providers, + combos: [ + { + id: "combo_opus", + name: "opus-route", + strategy: "fallback", + members: providers.map((provider, index) => ({ + providerId: provider.id, + model: `model-${index + 1}`, + })), + }, + ], + }); + const calls = []; + const router = createRouter({ + store, + logger: captureLogger(), + fetchImpl: async (url) => { + calls.push(new URL(url).hostname); + if (url.includes("provider-1")) { + return new Response(JSON.stringify({ error: { message: "rate limited" } }), { + status: 429, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.includes("provider-2")) { + return new Response( + "Failed to deserialize ChatCompletionRequestToolMessageContent", + { status: 400, headers: { "Content-Type": "text/plain" } } + ); + } + return successResponse("third member worked"); + }, + }); + + const result = await router.chatCompletions({ + body: { + model: "opus-route", + messages: [{ role: "user", content: "hello" }], + stream: false, + }, + }); + + assert.equal(result.ok, true, JSON.stringify(result.error)); + assert.equal(result.openAiJson.choices[0].message.content, "third member worked"); + assert.deepEqual(calls, ["provider-1.test", "provider-2.test", "provider-3.test"]); + }); + + it("reroutes a 2xx error payload and unusable 2xx body", async () => { + const store = createStore(tmpConfig()); + const providers = ["Error payload", "Malformed", "Usable"].map((name, index) => ({ + id: `prov_usable_${index + 1}`, + type: "openai-compat", + name, + baseUrl: `https://usable-${index + 1}.test/v1`, + apiKey: `usable-key-${index + 1}`, + enabled: true, + models: [{ id: `usable-model-${index + 1}`, name, enabled: true }], + })); + store.seed({ + providers, + combos: [ + { + id: "combo_usable", + strategy: "fallback", + members: providers.map((provider, index) => ({ + providerId: provider.id, + model: `usable-model-${index + 1}`, + })), + }, + ], + }); + const calls = []; + const router = createRouter({ + store, + logger: captureLogger(), + fetchImpl: async (url) => { + calls.push(new URL(url).hostname); + if (url.includes("usable-1")) { + return new Response(JSON.stringify({ error: { message: "failed inside a 200" } }), { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + if (url.includes("usable-2")) { + return new Response("not-json", { + status: 200, + headers: { "Content-Type": "application/json" }, + }); + } + return successResponse("usable response"); + }, + }); + + const result = await router.chatCompletions({ + body: { + model: "combo_usable", + messages: [{ role: "user", content: "hello" }], + stream: false, + }, + }); + + assert.equal(result.ok, true, JSON.stringify(result.error)); + assert.equal(result.openAiJson.choices[0].message.content, "usable response"); + assert.deepEqual(calls, ["usable-1.test", "usable-2.test", "usable-3.test"]); + }); + + it("reroutes immediate 2xx stream errors and empty streams before output starts", async () => { + const store = createStore(tmpConfig()); + const providers = ["Stream error", "Empty stream", "Usable stream"].map((name, index) => ({ + id: `prov_stream_${index + 1}`, + type: "openai-compat", + name, + baseUrl: `https://stream-${index + 1}.test/v1`, + apiKey: `stream-key-${index + 1}`, + enabled: true, + models: [{ id: `stream-model-${index + 1}`, name, enabled: true }], + })); + store.seed({ + providers, + combos: [ + { + id: "combo_stream_usable", + strategy: "fallback", + members: providers.map((provider, index) => ({ + providerId: provider.id, + model: `stream-model-${index + 1}`, + })), + }, + ], + }); + const calls = []; + const router = createRouter({ + store, + logger: captureLogger(), + fetchImpl: async (url) => { + calls.push(new URL(url).hostname); + if (url.includes("stream-1")) { + return new Response( + `event: error\ndata: ${JSON.stringify({ type: "error", error: { message: "provider failed" } })}\n\n`, + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ); + } + if (url.includes("stream-2")) { + return new Response("data: [DONE]\n\n", { + status: 200, + headers: { "Content-Type": "text/event-stream" }, + }); + } + return new Response( + [ + `data: ${JSON.stringify({ choices: [{ index: 0, delta: { content: "usable stream" } }] })}`, + "data: [DONE]", + "", + ].join("\n\n"), + { status: 200, headers: { "Content-Type": "text/event-stream" } } + ); + }, + }); + + const result = await router.chatCompletions({ + body: { + model: "combo_stream_usable", + messages: [{ role: "user", content: "hello" }], + stream: true, + }, + }); + const chunks = []; + await result.streamPipe({ write: (chunk) => chunks.push(String(chunk)) }); + + assert.equal(result.ok, true, JSON.stringify(result.error)); + assert.match(chunks.join(""), /usable stream/); + assert.deepEqual(calls, ["stream-1.test", "stream-2.test", "stream-3.test"]); + }); + it("records streaming usage after the final SSE event instead of an early zero-token success", async () => { const store = createStore(tmpConfig()); store.seed({ providers: [oauthAccount("prov_a", "token-a", 100)] });