diff --git a/README.md b/README.md index acc0c81..be13dab 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ **84% fewer tokens on JSON tool results. 53% fewer tokens on tool-heavy requests. Sub-300ms semantic cache hits. Zero code changes.** [![npm version](https://img.shields.io/npm/v/lynkr.svg)](https://www.npmjs.com/package/lynkr) -[![Tests](https://img.shields.io/badge/tests-1041%20passing-brightgreen)](https://github.com/Fast-Editor/Lynkr) +[![Tests](https://img.shields.io/badge/tests-1249%20passing-brightgreen)](https://github.com/Fast-Editor/Lynkr) [![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](LICENSE) [![Node.js](https://img.shields.io/badge/node-20%2B-green)](https://nodejs.org) [![Ask DeepWiki](https://deepwiki.com/badge.svg)](https://deepwiki.com/Fast-Editor/Lynkr) @@ -326,12 +326,45 @@ the local telemetry store: - **Routing accuracy** — over-/under-provisioned request counts, a self-audit of the tier router's decisions - **Request logs** — filterable by provider, tier, and errors, with latency, - tokens, and cost per request + tokens, and cost per request; every session id links to a drill-down - **Provider health** — configured providers, credential warnings, circuit breaker states +- **Insights** — analyzer findings ranked by *past overspend* (auditable + against spend already recorded, never projected savings): dead tool + schemas paying rent, harness side-traffic served on expensive tiers, + weak prompt-cache hit ratios, and Wilson-bounded tier-downsize verdicts. + Each finding ships its evidence rows, the fix, the caveat, and how the + figure was computed. "Not measured yet" is rendered distinctly from + zero — an un-run scan is not an all-clear. +- **Explore** — a Metric × Dimension (× Stack) pivot over routing telemetry + (spend/tokens/requests/sessions by provider, model, tier, request type, + day…) with preset views, period-over-period deltas, and CSV export +- **Session drill-down** — per-model cost mix and an input-context-per-call + chart where distillation and cache behavior are directly visible +- **Cache receipts** — projected switch/hold economics from cache-aware + routing next to the *measured* companion (dollars saved by cache reads + actually served) + +JSON APIs behind it +(`/dashboard/api/overview|usage|routing|logs|recommendations|analytics|sessions/:id|statusline`) +if you want the raw numbers. + +### Status line for Claude Code + +One line after every turn — routed tier, served model, today's spend, and +cache re-read share — with zero token cost (out-of-band, never enters model +context). Add to `~/.claude/settings.json`: + +```json +"statusLine": { "type": "command", "command": "lynkr-statusline" } +``` + +``` +◆ Opus 4.8 · MEDIUM → minimax-m3:cloud (ollama) · today $0.06 · cache 67% +``` -JSON APIs behind it (`/dashboard/api/overview|usage|routing|logs`) if you want -the raw numbers. +Always prints exactly one line and always exits 0 — a down proxy can never +break the harness. Set `LYNKR_PORT` if Lynkr isn't on 8081. ### Cost tracking & model pricing Per-request cost is computed from a model-pricing registry (LiteLLM → models.dev, @@ -551,7 +584,8 @@ With tier routing + token optimization: **additional 50-87% savings** on cloud p | **TOON JSON compression** | ✅ up to 87.6% | ❌ | ❌ | ❌ | | **Upstream SSE streaming** | ✅ native passthrough + cross-format transform | ⚠️ passthrough only | ✅ | ⚠️ | | **Semantic cache** | ✅ 171ms hits, 0 tokens | ❌ | ❌ | ✅ Prompt cache only | -| **Savings & routing dashboard** | ✅ spend, savings vs flagship, tier mix, routing accuracy, request logs | ⚠️ spend UI only | ⚠️ usage page | ✅ observability suite (no routing accuracy) | +| **Savings & routing dashboard** | ✅ spend, savings vs flagship, tier mix, routing accuracy, request logs, pivot explorer, session drill-down | ⚠️ spend UI only | ⚠️ usage page | ✅ observability suite (no routing accuracy) | +| **Cost recommendations** | ✅ evidence-backed findings (dead tool schemas, side-traffic, cache weak spots, Wilson-proven downsizes) ranked by past overspend | ❌ | ❌ | ⚠️ alerts, no findings | | **Long-term memory** | ✅ SQLite, per-session | ❌ | ❌ | ❌ | | **MCP integration** | ✅ | ❌ | ❌ | ❌ | | **Self-hosted** | ✅ Node.js only | ✅ Python stack | ❌ SaaS | ✅ Docker | diff --git a/bin/statusline.js b/bin/statusline.js new file mode 100755 index 0000000..0f3d196 --- /dev/null +++ b/bin/statusline.js @@ -0,0 +1,80 @@ +#!/usr/bin/env node +/** + * lynkr statusline — zero-token Claude Code status line. + * + * Wire into ~/.claude/settings.json: + * "statusLine": { "type": "command", "command": "lynkr-statusline" } + * + * Claude Code pipes session JSON on stdin after each turn. This command: + * - never enters model context (out-of-band — an in-loop MCP integration + * was measured at +36% model-weighted tokens by TokenJam; don't do that) + * - makes ONE local HTTP call to the Lynkr dashboard API with a hard + * 250ms timeout + * - always exits 0 and always prints exactly one line, no matter what + * fails — a broken status line must never break the harness. + * + * Output: ◆ · → ()[ · pin] · today $X · cache NN% + */ + +'use strict'; + +const PORT = Number(process.env.LYNKR_PORT) || Number(process.env.PORT) || 8081; +const TIMEOUT_MS = 250; + +function readStdin(maxMs) { + return new Promise((resolve) => { + let data = ''; + const timer = setTimeout(() => resolve(data), maxMs); + process.stdin.on('data', (c) => { data += c; }); + process.stdin.on('end', () => { clearTimeout(timer); resolve(data); }); + process.stdin.on('error', () => { clearTimeout(timer); resolve(data); }); + }); +} + +async function fetchStatus() { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), TIMEOUT_MS); + try { + const res = await fetch(`http://127.0.0.1:${PORT}/dashboard/api/statusline`, { + signal: controller.signal, + }); + if (!res.ok) return null; + return await res.json(); + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +async function main() { + const [stdinRaw, status] = await Promise.all([readStdin(150), fetchStatus()]); + + let clientModel = null; + try { + const parsed = JSON.parse(stdinRaw); + clientModel = parsed?.model?.display_name || parsed?.model?.id || null; + } catch { /* stdin absent or non-JSON — fine */ } + + const parts = []; + if (clientModel) parts.push(clientModel); + + if (status?.last) { + const served = `${status.last.tier || '?'} → ${status.last.model || '?'} (${status.last.provider || '?'})`; + parts.push(served + (status.last.pinned ? ' · pin' : '')); + } else { + parts.push('lynkr: no traffic yet'); + } + if (status && typeof status.todaySpendUsd === 'number') { + parts.push(`today $${status.todaySpendUsd.toFixed(2)}`); + } + if (status && typeof status.cacheReadPct === 'number') { + parts.push(`cache ${status.cacheReadPct}%`); + } + + process.stdout.write('◆ ' + parts.join(' · ') + '\n'); +} + +main() + .catch(() => { process.stdout.write('◆ lynkr statusline unavailable\n'); }) + .finally(() => process.exit(0)); diff --git a/documentation/token-optimization.md b/documentation/token-optimization.md index 455412a..be3beb9 100644 --- a/documentation/token-optimization.md +++ b/documentation/token-optimization.md @@ -289,6 +289,27 @@ curl http://localhost:8081/metrics | grep lynkr_tokens # lynkr_tokens_cached_total 500000 ``` +### Insights — where the remaining waste is + +The dashboard's **Insights** tab (`/dashboard/api/recommendations`) runs +analyzers over routing telemetry and reports *past overspend* — figures you +can check against spend already recorded, never projections: + +- **deadweight** — sessions that carried tool schemas on every request and + never called a tool (schema rent, priced per session) +- **side-requests** — harness bookkeeping (title generation, topic + detection) served above the SIMPLE tier +- **cache-weakspots** — prompt-cache hit ratio per (provider, model), from + per-request cache counters; weak pairs priced as an explicit ceiling +- **downsize** — request types where the tier below has statistically + proven itself (Wilson 95% lower bound ≥ 0.7 on ≥ 20 quality-scored + samples — a raw 70% average on a small sample stays "unproven") + +Each finding carries its evidence rows, the fix, a caveat, and how the +figure was computed. The Routing tab additionally shows the **measured** +cache companion — dollars saved by cache reads actually served — beside the +projected switch/hold receipts from cache-aware routing. + ### Per-Request Logging ```bash diff --git a/package.json b/package.json index f82fd67..12286ce 100644 --- a/package.json +++ b/package.json @@ -25,7 +25,8 @@ "main": "index.js", "bin": { "lynkr": "bin/cli.js", - "lynkr-setup": "scripts/setup.js" + "lynkr-setup": "scripts/setup.js", + "lynkr-statusline": "bin/statusline.js" }, "scripts": { "postinstall": "node scripts/check-native.js", @@ -36,7 +37,7 @@ "dev": "nodemon index.js", "lint": "eslint src index.js", "test": "npm run test:unit && npm run test:performance", - "test:unit": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com LOG_FILE_ENABLED=false node --test test/routing.test.js test/hybrid-routing-integration.test.js test/retry-logic.test.js test/sse-transformer.test.js test/passthrough-stream.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/toon-compression.test.js test/gcf-compression.test.js test/llamacpp-integration.test.js test/resilience.test.js test/telemetry-routing.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js test/distill.test.js test/large-payload.test.js test/prompt-cache-injection.test.js test/risk-analyzer.test.js test/interaction-block.test.js test/preflight.test.js test/token-reduction.test.js test/session-affinity.test.js test/cache-state.test.js test/cache-switch-cost.test.js test/model-registry-cost.test.js test/output-format-guard.test.js test/tier-fallback.test.js test/wrap.test.js test/init.test.js test/tool-call-response-metadata.test.js test/degradation.test.js test/routing-telemetry-columns.test.js test/sticky-routing.test.js test/knn-ambiguous-escalate.test.js test/deescalator.test.js test/client-profiles.test.js test/strip-internal-fields.test.js test/complexity-tool-subtraction.test.js test/bandit.test.js test/routing-propensity.test.js test/reward-pipeline.test.js test/knn-cold-start.test.js test/calibration.test.js test/feedback-loop.test.js test/session-fingerprint.test.js test/side-request-guards.test.js test/verifier.test.js test/intent-score.test.js test/difficulty-classifier.test.js test/classifier-setup.test.js test/usage-stats.test.js test/loop-guard.test.js test/moonshot-model-mapping.test.js", + "test:unit": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com LOG_FILE_ENABLED=false node --test test/routing.test.js test/hybrid-routing-integration.test.js test/retry-logic.test.js test/sse-transformer.test.js test/passthrough-stream.test.js test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js test/azure-openai-config.test.js test/azure-openai-format-conversion.test.js test/azure-openai-routing.test.js test/azure-openai-streaming.test.js test/azure-openai-error-resilience.test.js test/azure-openai-integration.test.js test/openai-integration.test.js test/toon-compression.test.js test/gcf-compression.test.js test/llamacpp-integration.test.js test/resilience.test.js test/telemetry-routing.test.js test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js test/distill.test.js test/large-payload.test.js test/prompt-cache-injection.test.js test/risk-analyzer.test.js test/interaction-block.test.js test/preflight.test.js test/token-reduction.test.js test/session-affinity.test.js test/cache-state.test.js test/cache-switch-cost.test.js test/lens-recommendations.test.js test/model-registry-cost.test.js test/output-format-guard.test.js test/tier-fallback.test.js test/wrap.test.js test/init.test.js test/tool-call-response-metadata.test.js test/degradation.test.js test/routing-telemetry-columns.test.js test/sticky-routing.test.js test/knn-ambiguous-escalate.test.js test/deescalator.test.js test/client-profiles.test.js test/strip-internal-fields.test.js test/complexity-tool-subtraction.test.js test/bandit.test.js test/routing-propensity.test.js test/reward-pipeline.test.js test/knn-cold-start.test.js test/calibration.test.js test/feedback-loop.test.js test/session-fingerprint.test.js test/side-request-guards.test.js test/verifier.test.js test/intent-score.test.js test/difficulty-classifier.test.js test/classifier-setup.test.js test/usage-stats.test.js test/loop-guard.test.js test/moonshot-model-mapping.test.js", "test:memory": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/memory/store.test.js test/memory/surprise.test.js test/memory/extractor.test.js test/memory/search.test.js test/memory/retriever.test.js test/memory/distiller.test.js test/memory/distiller-freeze.test.js test/memory/wiki.test.js test/memory/skills-cache.test.js test/memory/tencentdb-launcher.test.js", "test:new-features": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node --test test/passthrough-mode.test.js test/openrouter-error-resilience.test.js test/format-conversion.test.js", "test:performance": "DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/hybrid-routing-performance.test.js && DATABRICKS_API_KEY=test-key DATABRICKS_API_BASE=http://test.com node test/performance-tests.js", diff --git a/public/dashboard.html b/public/dashboard.html index d4a1e3b..01ebdb9 100644 --- a/public/dashboard.html +++ b/public/dashboard.html @@ -50,6 +50,14 @@ class="ring-tab px-4 py-2 rounded-md text-sm font-medium text-slate-400 hover:text-white hover:bg-slate-700"> Logs + +
@@ -137,13 +145,18 @@ data: {}, usageWindow: '7d', logFilters: { provider: '', tier: '', error: false }, + insightOpen: null, + sessionId: null, + explore: { metric: 'spend', by: 'tier', stack: '', days: 7 }, _refreshTimer: null, _countdownTimer: null, _countdown: 30, _charts: {}, init() { - const hash = location.hash.slice(1) || 'overview'; + let hash = location.hash.slice(1) || 'overview'; + // Session detail needs in-memory state — a cold load on that hash has none. + if (hash === 'session' && !this.sessionId) hash = 'logs'; this.navigate(hash, true); this._startCountdown(); }, @@ -198,6 +211,14 @@ if (this.logFilters.error) q.set('error', 'true'); data = await this._fetch('/dashboard/api/logs?' + q); } + if (this.page === 'insights') data = await this._fetch('/dashboard/api/recommendations'); + if (this.page === 'explore') { + const e = this.explore; + const q = new URLSearchParams({ metric: e.metric, by: e.by, days: e.days }); + if (e.stack) q.set('stack', e.stack); + data = await this._fetch('/dashboard/api/analytics?' + q); + } + if (this.page === 'session') data = await this._fetch('/dashboard/api/sessions/' + encodeURIComponent(this.sessionId)); this.data[this.page] = data; this._render(data); @@ -228,6 +249,9 @@ usage: () => this._renderUsage(data), routing: () => this._renderRouting(data), logs: () => this._renderLogs(data), + insights: () => this._renderInsights(data), + explore: () => this._renderExplore(data), + session: () => this._renderSession(data), }[this.page]?.() || ''; document.getElementById('content').innerHTML = `
${html}
`; this._afterRender(data); @@ -520,6 +544,13 @@

Cache Economics (last ${d.

` : emptyState('No gated switch decisions yet — appears once a pinned session hits a downgrade decision')} +
+ Measured (past tense, auditable) + ${d.cacheMeasured && d.cacheMeasured.rowsMeasured > 0 ? ` + ${fmt.usd2(d.cacheMeasured.measuredSavedUsd)} saved by cache reads actually served + ${fmt.num(d.cacheMeasured.rowsMeasured)} requests measured · weakest: ${d.cacheMeasured.perModel[0] ? `${d.cacheMeasured.perModel[0].model} at ${d.cacheMeasured.perModel[0].hitRatio != null ? Math.round(d.cacheMeasured.perModel[0].hitRatio * 100) + '%' : '—'}` : '—'} + ` : `not measured yet — cache counters accrue from this version onward (not the same as zero savings)`} +
`, 'mb-6')} @@ -589,6 +620,7 @@

Circuit Breakers

${r.error_type||'—'} ${r.cost_usd!=null?fmt.usd(r.cost_usd):'—'} ${r.was_fallback?'fallback':'—'} + ${r.session_id ? `` : '—'} `).join(''); return ` @@ -632,6 +664,7 @@

Request Logs

Error Cost Flags + Session ${tableRows} @@ -645,6 +678,297 @@

Request Logs

if (this.page === 'usage' && data?.daily?.length && data.totals?.requests > 0) { this._drawUsageChart(data.daily); } + if (this.page === 'explore' && data?.rows?.length) this._drawExploreChart(data); + if (this.page === 'session' && data?.contextSeries?.length) this._drawContextChart(data.contextSeries); + }, + + /* ── INSIGHTS (Lens recommendations) ─────────── */ + openInsight(id) { + this.insightOpen = this.insightOpen === id ? null : id; + this._render(this.data.insights); + }, + + showSession(id) { + if (!id) return; + this.sessionId = id; + this.navigate('session'); + }, + + _renderInsights(d) { + if (!d) return emptyState('No data'); + const f = d.findings || []; + const actionable = f.filter(x => x.state === 'actionable'); + const sharePct = d.spendUsd > 0 ? Math.min(100, (d.totalRecoverableUsd / d.spendUsd) * 100) : 0; + const TONES = ['bg-amber-500','bg-blue-500','bg-purple-500','bg-green-500','bg-rose-500']; + + // Hero — one netted, backward-looking figure. Past tense on purpose. + const segments = actionable.filter(x => x.pastOverspendUsd > 0); + const segTotal = segments.reduce((a, x) => a + x.pastOverspendUsd, 0) || 1; + const shareBar = segments.length ? ` +
+ ${segments.map((x, i) => `
`).join('')} +
+
+ ${segments.map((x, i) => `${x.title}`).join('')} +
` : ''; + + const hero = card(` +

Past overspend found (last ${Math.round(d.windowMs / 86400000)}d)

+
+ ${fmt.usd2(d.totalRecoverableUsd)} + of ${fmt.usd2(d.spendUsd)} spent · ${sharePct.toFixed(1)}% +
+ ${shareBar} +

Figures are backward-looking — checkable against spend already recorded. Ceilings are labeled in each finding's basis.

+ `, 'mb-6'); + + const mechBadge = m => ({ + auto: 'lynkr can act', + snippet: 'client-side fix', + info: 'diagnosis', + }[m] || ''); + const stateBadge = s => ({ + actionable: '', + clean: 'ran — found nothing', + not_measured: 'not measured yet', + }[s] || ''); + + const tiles = `
+ ${f.map(x => ` +
+
+ ${x.title} + ${mechBadge(x.mechanism)} +
+ ${x.state === 'actionable' + ? `

${typeof x.pastOverspendUsd === 'number' ? fmt.usd2(x.pastOverspendUsd) : fmt.tok(x.pastOverspendTokens || 0) + ' tok'}

+

click for evidence

` + : stateBadge(x.state)} +
`).join('')} +
`; + + // Detail shell — Evidence / Fix / Caveat / How computed. + let detail = ''; + const open = f.find(x => x.id === this.insightOpen); + if (open) { + const evid = open.evidence?.rows?.length ? ` +
+ + + ${open.evidence.columns.map(c => ``).join('')} + + ${open.evidence.rows.map(r => ` + + ${r.map(v => ``).join('')} + `).join('')} + +
${c}
${typeof v === 'number' && !Number.isInteger(v) ? fmt.usd(v) : v}
+
` : emptyState('No evidence rows'); + detail = card(` +
+

${open.title}

+
${(open.stats || []).map(s => `${fmt.num(s.value)} ${s.label}`).join('')}
+
+
+
+

Evidence — where this figure came from

+ ${evid} +
+
+ ${open.fix ? `

The fix

${open.fix}

` : ''} + ${open.caveat ? `

Caveat

${open.caveat}

` : ''} +

How this figure is computed

${open.estimateBasis || '—'}

+
+
+ `, 'mb-6'); + } + + return ` +

Insights

+ ${hero} + ${tiles} + ${detail} + `; + }, + + /* ── EXPLORE (pivot explorer) ────────────────── */ + setExplore(key, value) { + this.explore[key] = key === 'days' ? parseInt(value, 10) : value; + this.refresh(); + }, + applyPreset(metric, by, stack) { + this.explore = { ...this.explore, metric, by, stack }; + this.refresh(); + }, + exportCsv() { + const d = this.data.explore; + if (!d?.rows?.length) return; + const header = d.stack ? 'dimension,stack,value' : 'dimension,value'; + const lines = d.rows.map(r => d.stack + ? `"${r.dim}","${r.stack}",${r.value}` + : `"${r.dim}",${r.value}`); + const blob = new Blob([header + '\n' + lines.join('\n')], { type: 'text/csv' }); + const a = document.createElement('a'); + a.href = URL.createObjectURL(blob); + a.download = `lynkr-${d.metric}-by-${d.by}.csv`; + a.click(); + }, + + _renderExplore(d) { + if (!d) return emptyState('No data'); + const e = this.explore; + const METRICS = ['spend', 'tokens', 'requests', 'sessions']; + const DIMS = ['provider', 'model', 'tier', 'request_type', 'routing_method', 'switch_reason', 'day']; + const sel = (id, opts, val, allowEmpty) => ` + `; + + const kpiVal = { spend: fmt.usd2(d.kpis?.spend), tokens: fmt.tok(d.kpis?.tokens || 0), requests: fmt.num(d.kpis?.requests), sessions: fmt.num(d.kpis?.sessions) }; + const kpiDelta = k => { + if (!d.kpiDeltas) return 'prior window too thin to compare'; + const v = d.kpiDeltas[k]; + const sign = v > 0 ? '+' : ''; + const cls = v > 0 ? 'text-amber-400' : 'text-green-400'; + const fmtV = k === 'spend' ? fmt.usd2(Math.abs(v)) : k === 'tokens' ? fmt.tok(Math.abs(v)) : fmt.num(Math.abs(v)); + return `${sign === '+' ? '▲' : '▼'} ${fmtV} vs prior`; + }; + const kpis = `
+ ${METRICS.map(k => ` +
+

${k}

+

${kpiVal[k] ?? '—'}

+ ${kpiDelta(k)} +
`).join('')} +
`; + + const presets = [ + ['Spend by tier', 'spend', 'tier', ''], + ['Tokens by model', 'tokens', 'model', ''], + ['Requests by type', 'requests', 'request_type', ''], + ['Spend over days by tier', 'spend', 'day', 'tier'], + ]; + + return ` +
+

Explore

+ +
+ ${card(` +
+ ${sel('metric', METRICS, e.metric)} + by + ${sel('by', DIMS, e.by)} + stack + ${sel('stack', DIMS, e.stack, true)} + ${sel('days', [1, 7, 30, 90], e.days)} + days +
+
+ ${presets.map(([label, m, b, s]) => ` + `).join('')} +
+ `, 'mb-6')} + ${kpis} + ${d.rows?.length + ? card(`
`) + : emptyState('No rows for this pivot in the window — not the same as “nothing happened”: check the window size.')} + `; + }, + + _drawExploreChart(d) { + const canvas = document.getElementById('explore-chart'); + if (!canvas || typeof Chart === 'undefined') return; + const PALETTE = ['rgba(34,197,94,0.8)','rgba(59,130,246,0.8)','rgba(245,158,11,0.8)','rgba(168,85,247,0.8)','rgba(244,63,94,0.8)','rgba(14,165,233,0.8)','rgba(148,163,184,0.8)']; + let cfg; + if (d.stack) { + const dims = [...new Set(d.rows.map(r => r.dim))]; + const stacks = [...new Set(d.rows.map(r => r.stack))]; + cfg = { + type: 'bar', + data: { + labels: dims, + datasets: stacks.map((s, i) => ({ + label: String(s ?? '—'), + data: dims.map(dim => d.rows.find(r => r.dim === dim && r.stack === s)?.value ?? 0), + backgroundColor: PALETTE[i % PALETTE.length], + })), + }, + options: { responsive: true, maintainAspectRatio: false, scales: { x: { stacked: true }, y: { stacked: true } } }, + }; + } else { + const rows = d.rows.slice(0, 20); + cfg = { + type: 'bar', + data: { + labels: rows.map(r => String(r.dim ?? '—')), + datasets: [{ label: d.metric, data: rows.map(r => r.value), backgroundColor: PALETTE[1] }], + }, + options: { responsive: true, maintainAspectRatio: false, indexAxis: rows.length > 8 ? 'y' : 'x', plugins: { legend: { display: false } } }, + }; + } + this._charts.explore = new Chart(canvas, cfg); + }, + + /* ── SESSION DETAIL ──────────────────────────── */ + _renderSession(d) { + if (!d) return emptyState('Session not found'); + const dur = d.lastSeen - d.firstSeen; + const modelRows = d.models.map(m => ` + + ${m.provider} + ${m.model || '—'} + ${fmt.num(m.requests)} + ${fmt.tok(m.inputTokens)} + ${fmt.tok(m.outputTokens)} + ${fmt.tok(m.cacheReadTokens)} + ${fmt.usd(m.costUsd)} + `).join(''); + + return ` +
+ +

Session

+ ${d.sessionId} +
+
+ ${[['requests', fmt.num(d.requests)], ['spend', fmt.usd(d.totalCostUsd)], ['tool calls', fmt.num(d.toolCalls)], ['duration', fmt.ms(dur)]] + .map(([l, v]) => `

${l}

${v}

`).join('')} +
+ ${card(` +

Input (context) tokens per call — distillation and cache behavior are visible here

+
+ `, 'mb-6')} + ${card(` +

Model mix

+
+ + + + ${modelRows} +
providermodelrequestsinputoutputcache readscost
+ `)} + `; + }, + + _drawContextChart(series) { + const canvas = document.getElementById('session-context-chart'); + if (!canvas || typeof Chart === 'undefined') return; + this._charts.sessionCtx = new Chart(canvas, { + type: 'bar', + data: { + labels: series.map((_, i) => i + 1), + datasets: [ + { label: 'input tokens', data: series.map(p => p.inputTokens), backgroundColor: 'rgba(59,130,246,0.8)' }, + { label: 'cache reads', data: series.map(p => p.cacheReadTokens ?? 0), backgroundColor: 'rgba(34,197,94,0.6)' }, + ], + }, + options: { responsive: true, maintainAspectRatio: false, scales: { x: { title: { display: true, text: 'call #' } } } }, + }); }, _drawUsageChart(daily) { diff --git a/src/clients/databricks.js b/src/clients/databricks.js index 1f5b3fc..b874ff9 100644 --- a/src/clients/databricks.js +++ b/src/clients/databricks.js @@ -3000,6 +3000,8 @@ async function invokeModel(body, options = {}) { pinned: routingResult.pinned ? 1 : 0, switch_reason: routingResult.switch_reason ?? null, cache_decision: routingResult._cacheDecision ?? null, + cache_read_tokens: result.json?.usage?.cache_read_input_tokens ?? null, + cache_creation_tokens: result.json?.usage?.cache_creation_input_tokens ?? null, }); // WS5.4 — feedback loop (success path). @@ -3351,6 +3353,8 @@ async function invokeModel(body, options = {}) { pinned: routingResult.pinned ? 1 : 0, switch_reason: routingResult.switch_reason ?? null, cache_decision: routingResult._cacheDecision ?? null, + cache_read_tokens: fallbackResult.json?.usage?.cache_read_input_tokens ?? null, + cache_creation_tokens: fallbackResult.json?.usage?.cache_creation_input_tokens ?? null, }); // WS5.4 — feedback loop (fallback success). The served provider diff --git a/src/dashboard/api.js b/src/dashboard/api.js index 45104cf..a87b139 100644 --- a/src/dashboard/api.js +++ b/src/dashboard/api.js @@ -208,8 +208,12 @@ function routing(req, res) { // Cache-aware routing (Phase 6): per-decision switch/hold economics, // aggregated into "cache dollars saved by routing". const cacheEconomics = telemetry.getCacheEconomics({ since }); + // Lens money-framing: the MEASURED companion — cache reads actually + // served this window, auditable against the bill (vs the projected + // figures above). null until cache counters have accrued. + const cacheMeasured = telemetry.getMeasuredCacheSavings({ since }); - res.json({ tierDefinitions: TIER_DEFINITIONS, accuracy, stats, providerStats, circuitBreakers: cbStates, cacheEconomics, window: win.label }); + res.json({ tierDefinitions: TIER_DEFINITIONS, accuracy, stats, providerStats, circuitBreakers: cbStates, cacheEconomics, cacheMeasured, window: win.label }); } catch (e) { res.status(500).json({ error: 'routing_api_error', detail: e.message }); } @@ -233,4 +237,76 @@ function logs(req, res) { } } -module.exports = { overview, usage, routing, logs }; +/* ── Lens endpoints (feature/lens-dashboard) ─────────────────────────── */ + +// Recommendations — analyzer findings ranked by past overspend. +function recommendations(req, res) { + try { + const engine = require('./recommendations'); + const windowMs = req.query.window === '30d' ? 30 * 86400000 + : req.query.window === '24h' ? 86400000 + : 7 * 86400000; + res.json(engine.run({ windowMs, force: req.query.force === 'true' })); + } catch (e) { + res.status(500).json({ error: 'recommendations_api_error', detail: e.message }); + } +} + +// Session drill-down: per-model mix + context-growth series + raw rows. +function sessionDetail(req, res) { + try { + const detail = telemetry.getSessionDetail(req.params.id); + if (!detail) return res.status(404).json({ error: 'session_not_found' }); + res.json(detail); + } catch (e) { + res.status(500).json({ error: 'session_api_error', detail: e.message }); + } +} + +// Pivot explorer: metric × dimension (× stack), whitelisted server-side. +function analytics(req, res) { + try { + const days = Math.min(90, Math.max(1, parseInt(req.query.days || '7', 10) || 7)); + const data = telemetry.getAnalytics({ + metric: req.query.metric, + by: req.query.by, + stack: req.query.stack || null, + since: Date.now() - days * 86400000, + }); + if (!data) return res.status(503).json({ error: 'telemetry_unavailable' }); + res.json(data); + } catch (e) { + res.status(500).json({ error: 'analytics_api_error', detail: e.message }); + } +} + +// Statusline: one cheap snapshot for the zero-token status line — last +// routed request, today's spend, measured cache re-read share. +function statusline(req, res) { + try { + const last = telemetry.query({ limit: 1 })[0] ?? null; + const since = Date.now() - 86400000; + const db = telemetry.getDb(); + let today = null; + if (db) { + today = db.prepare( + `SELECT SUM(COALESCE(cost_usd,0)) spend, + SUM(COALESCE(cache_read_tokens,0)) rd, + SUM(COALESCE(cache_read_tokens,0)+COALESCE(cache_creation_tokens,0)+COALESCE(input_tokens,0)) total + FROM routing_telemetry WHERE timestamp > ?` + ).get(since); + } + res.json({ + last: last && { + tier: last.tier, provider: last.provider, model: last.model, + pinned: !!last.pinned, at: last.timestamp, + }, + todaySpendUsd: today?.spend ?? null, + cacheReadPct: today && today.total > 0 ? Math.round((today.rd / today.total) * 100) : null, + }); + } catch (e) { + res.status(500).json({ error: 'statusline_api_error', detail: e.message }); + } +} + +module.exports = { overview, usage, routing, logs, recommendations, sessionDetail, analytics, statusline }; diff --git a/src/dashboard/recommendations/cache-weakspots.js b/src/dashboard/recommendations/cache-weakspots.js new file mode 100644 index 0000000..0b2fcec --- /dev/null +++ b/src/dashboard/recommendations/cache-weakspots.js @@ -0,0 +1,88 @@ +/** + * cache-weakspots — prompt-cache hit ratio per (provider, model), worst first. + * + * Reads the per-request cache counters (cache_read_tokens / + * cache_creation_tokens). Rows recorded before capture began are NULL and + * excluded — when nothing has been measured yet the finding says so + * explicitly instead of reporting a fake all-clear. + */ + +const MIN_REQUESTS = 5; +const WEAK_RATIO = 0.5; + +module.exports = { + id: 'cache-weakspots', + title: 'Weak prompt-cache hit ratios', + + analyze({ db, since }) { + const rows = db + .prepare( + `SELECT provider, model, COUNT(*) n, + SUM(COALESCE(cache_read_tokens,0)) rd, + SUM(COALESCE(cache_creation_tokens,0)) cr, + SUM(COALESCE(input_tokens,0)) inp + FROM routing_telemetry + WHERE timestamp > ? AND cache_read_tokens IS NOT NULL + GROUP BY provider, model + HAVING COUNT(*) >= ${MIN_REQUESTS}` + ) + .all(since); + + if (!rows.length) { + return { + id: this.id, title: this.title, mechanism: 'info', state: 'not_measured', + framing: 'usd', pastOverspendUsd: null, pastOverspendTokens: null, + stats: [], evidence: { columns: [], rows: [] }, fix: null, + caveat: null, + estimateBasis: 'Cache counters are captured from this version onward — data accrues as requests flow. An un-run measurement is not an all-clear.', + }; + } + + const { resolveCacheEconomics } = require('../../routing/cache-economics'); + const weak = []; + let usd = 0; + for (const r of rows) { + const total = r.rd + r.cr + r.inp; + const ratio = total > 0 ? r.rd / total : 0; + if (ratio >= WEAK_RATIO) continue; + const econ = resolveCacheEconomics(r.provider, r.model); + // Ceiling: what the uncached input cost beyond cache-read price. + const missedUsd = econ.unknownPricing || econ.mechanism === 'local' + ? null + : (r.inp * Math.max(0, econ.inputPerM - econ.cacheReadPerM)) / 1_000_000; + if (missedUsd != null) usd += missedUsd; + weak.push([r.provider, r.model, r.n, `${Math.round(ratio * 100)}%`, r.inp, missedUsd ?? '(local/unknown)']); + } + + if (!weak.length) { + return { + id: this.id, title: this.title, mechanism: 'info', state: 'clean', + framing: 'usd', pastOverspendUsd: 0, pastOverspendTokens: null, + stats: [{ label: 'models measured', value: rows.length }], + evidence: { columns: [], rows: [] }, fix: null, caveat: null, + estimateBasis: `Hit ratio = cache reads ÷ (reads + writes + uncached input); weak = below ${WEAK_RATIO * 100}%.`, + }; + } + + return { + id: this.id, + title: this.title, + mechanism: 'info', + state: 'actionable', + framing: 'usd', + pastOverspendUsd: usd, + pastOverspendTokens: null, + stats: [ + { label: 'weak (provider, model) pairs', value: weak.length }, + { label: 'models measured', value: rows.length }, + ], + evidence: { + columns: ['provider', 'model', 'requests', 'hit ratio', 'uncached input tokens', 'ceiling $'], + rows: weak.sort((a, b) => parseFloat(a[3]) - parseFloat(b[3])), + }, + fix: 'Low ratios usually mean an unstable prefix: per-request bytes in the system prompt, client-side history rewrites, or conversations too short to cache. Check the silent-invalidator list and whether the distiller freeze covers these sessions.', + caveat: 'The dollar figure is a CEILING — it prices every uncached input token at the read discount, which assumes perfect cacheability. Real recovery is lower.', + estimateBasis: 'Uncached input tokens × (input price − cache-read price) for pairs below a 50% hit ratio, this window. NULL-counter rows (pre-capture) excluded.', + }; + }, +}; diff --git a/src/dashboard/recommendations/deadweight.js b/src/dashboard/recommendations/deadweight.js new file mode 100644 index 0000000..4bb03f9 --- /dev/null +++ b/src/dashboard/recommendations/deadweight.js @@ -0,0 +1,82 @@ +/** + * deadweight — tool schemas carried on every request but never called. + * + * Telemetry stores per-request tool COUNTS, not names, so the detectable + * unit is the session: sessions whose every request shipped a tool loadout + * and whose tool_calls_made never left zero paid schema rent for nothing. + * (Per-tool attribution needs tool names in telemetry — future work.) + */ + +const EST_TOKENS_PER_TOOL = 150; // typical JSON schema; stated in estimateBasis +const MIN_REQUESTS = 3; + +module.exports = { + id: 'deadweight', + title: 'Tool schemas carried but never used', + + analyze({ db, since }) { + const rows = db + .prepare( + `SELECT session_id, + COUNT(*) reqs, + AVG(COALESCE(tool_count,0)) avg_tools, + MAX(model) model, + MAX(provider) provider, + SUM(COALESCE(cost_usd,0)) session_cost + FROM routing_telemetry + WHERE timestamp > ? AND session_id IS NOT NULL AND COALESCE(tool_count,0) > 0 + GROUP BY session_id + HAVING SUM(COALESCE(tool_calls_made,0)) = 0 AND COUNT(*) >= ${MIN_REQUESTS} + ORDER BY reqs * avg_tools DESC + LIMIT 50` + ) + .all(since); + + if (!rows.length) { + return { + id: this.id, title: this.title, mechanism: 'snippet', state: 'clean', + framing: 'usd', pastOverspendUsd: 0, pastOverspendTokens: 0, + stats: [], evidence: { columns: [], rows: [] }, + fix: null, caveat: null, + estimateBasis: 'Sessions with >=3 requests, a tool loadout on every request, and zero tool calls.', + }; + } + + const { resolveCacheEconomics } = require('../../routing/cache-economics'); + let tokens = 0; + let usd = 0; + const evidence = []; + for (const r of rows) { + const wastedTokens = Math.round(r.avg_tools * EST_TOKENS_PER_TOOL * r.reqs); + tokens += wastedTokens; + const econ = resolveCacheEconomics(r.provider, r.model); + const rowUsd = econ.unknownPricing ? null : (wastedTokens * econ.inputPerM) / 1_000_000; + if (rowUsd != null) usd += rowUsd; + evidence.push([ + r.session_id, r.reqs, Math.round(r.avg_tools), wastedTokens, + rowUsd != null ? rowUsd : '(price unknown)', + ]); + } + + return { + id: this.id, + title: this.title, + mechanism: 'snippet', + state: 'actionable', + framing: 'usd', + pastOverspendUsd: usd, + pastOverspendTokens: tokens, + stats: [ + { label: 'sessions affected', value: rows.length }, + { label: 'est. schema tokens re-sent', value: tokens }, + ], + evidence: { + columns: ['session', 'requests', 'avg tools', 'est. wasted tokens', 'est. $'], + rows: evidence, + }, + fix: 'These sessions never called a tool — their MCP servers/tool registrations paid schema rent on every request. Trim unused MCP servers from the client config, or scope them per-project.', + caveat: 'Schema size is estimated at ~150 tokens per tool; telemetry stores tool counts, not names, so attribution is per-session rather than per-tool.', + estimateBasis: 'avg tools × ~150 est. tokens/schema × requests per session, priced at each session\'s model input rate. Sessions with any tool call are excluded entirely.', + }; + }, +}; diff --git a/src/dashboard/recommendations/downsize.js b/src/dashboard/recommendations/downsize.js new file mode 100644 index 0000000..fc5032a --- /dev/null +++ b/src/dashboard/recommendations/downsize.js @@ -0,0 +1,109 @@ +/** + * downsize — request types where the tier below has statistically proven + * itself, Wilson-bounded. + * + * The de-escalator's live rule uses raw averages (>=30 rows, avg quality + * >=70). This analyzer applies the stricter test TokenJam-bench uses for + * verdicts: the WILSON LOWER BOUND of the lower tier's success rate must + * clear the bar, so 21-of-30 (lower bound ~0.52) stays "unproven" while + * 140-of-170 (~0.76) is a verdict. Success = quality >= 70 and no error. + * + * No quality-equivalence claim is made — the verdict is "the lower tier's + * measured floor clears the bar on this request type", nothing more. + */ + +const { wilsonLowerBound } = require('./wilson'); + +const TIER_ORDER = ['SIMPLE', 'MEDIUM', 'COMPLEX', 'REASONING']; +const MIN_SAMPLES = 20; +const SUCCESS_QUALITY = 70; +const WILSON_BAR = 0.7; +const EVIDENCE_WINDOW_MS = 30 * 24 * 60 * 60 * 1000; // evidence looks back further than the spend window + +module.exports = { + id: 'downsize', + title: 'Tiers that could step down (Wilson-proven)', + + analyze({ db, since, until }) { + // Lower-tier track record per request_type over 30 days. + const evidence = db + .prepare( + `SELECT tier, request_type, + COUNT(*) n, + SUM(CASE WHEN COALESCE(quality_score,0) >= ${SUCCESS_QUALITY} AND error_type IS NULL THEN 1 ELSE 0 END) k, + AVG(COALESCE(cost_usd,0)) avg_cost + FROM routing_telemetry + WHERE timestamp > ? AND tier IS NOT NULL AND request_type IS NOT NULL + GROUP BY tier, request_type` + ) + .all(until - EVIDENCE_WINDOW_MS); + + const byKey = new Map(evidence.map((r) => [`${r.tier}::${r.request_type}`, r])); + + // Spend at each upper tier per request_type, this window. + const upperSpend = db + .prepare( + `SELECT tier, request_type, COUNT(*) n, SUM(COALESCE(cost_usd,0)) cost, AVG(COALESCE(cost_usd,0)) avg_cost + FROM routing_telemetry + WHERE timestamp > ? AND tier IS NOT NULL AND request_type IS NOT NULL + GROUP BY tier, request_type` + ) + .all(since); + + const rows = []; + let usd = 0; + let anyEvidence = false; + for (const u of upperSpend) { + const idx = TIER_ORDER.indexOf(u.tier); + if (idx <= 0) continue; + const lower = byKey.get(`${TIER_ORDER[idx - 1]}::${u.request_type}`); + if (!lower) continue; + anyEvidence = true; + if (lower.n < MIN_SAMPLES) continue; + const bound = wilsonLowerBound(lower.k, lower.n); + if (bound < WILSON_BAR) continue; + // Empirical per-request price delta between the two tiers on the SAME + // request type — measured, not modeled. + const delta = Math.max(0, (u.avg_cost ?? 0) - (lower.avg_cost ?? 0)) * u.n; + usd += delta; + rows.push([ + u.request_type, u.tier, `${TIER_ORDER[idx - 1]}`, + `${lower.k}/${lower.n}`, bound.toFixed(2), u.n, delta, + ]); + } + + if (!rows.length) { + return { + id: this.id, title: this.title, mechanism: 'info', + state: anyEvidence ? 'clean' : 'not_measured', + framing: 'usd', pastOverspendUsd: anyEvidence ? 0 : null, pastOverspendTokens: null, + stats: [], + evidence: { columns: [], rows: [] }, fix: null, caveat: null, + estimateBasis: anyEvidence + ? `No (tier, request_type) pair has a lower tier whose Wilson lower bound clears ${WILSON_BAR} on >=${MIN_SAMPLES} samples yet.` + : 'Needs quality-scored telemetry with request_type populated — accrues as traffic flows (request_type capture is recent).', + }; + } + + rows.sort((a, b) => b[6] - a[6]); + return { + id: this.id, + title: this.title, + mechanism: 'info', + state: 'actionable', + framing: 'usd', + pastOverspendUsd: usd, + pastOverspendTokens: null, + stats: [ + { label: 'proven (request_type, tier) pairs', value: rows.length }, + ], + evidence: { + columns: ['request_type', 'served tier', 'proven lower tier', 'lower success', 'Wilson lower bound', 'requests this window', 'measured delta $'], + rows, + }, + fix: 'The live de-escalator will demote these automatically as its own thresholds clear; to act sooner, adjust the TIER_* mapping for these request types or lower the de-escalator sample bar deliberately.', + caveat: 'Verdict is "the lower tier\'s measured success floor clears the bar" — not quality equivalence. Success = quality >= 70 with no error.', + estimateBasis: 'Per-request cost delta between tiers measured on the SAME request_type (30-day evidence window), × this window\'s upper-tier request count. Wilson 95% lower bound.', + }; + }, +}; diff --git a/src/dashboard/recommendations/index.js b/src/dashboard/recommendations/index.js new file mode 100644 index 0000000..078a126 --- /dev/null +++ b/src/dashboard/recommendations/index.js @@ -0,0 +1,106 @@ +/** + * Lens recommendations engine. + * + * Registry-driven analyzers over routing telemetry that answer "what should + * the operator change", TokenJam-style, with three disciplines borrowed + * deliberately: + * + * - PAST OVERSPEND, never projected savings: every dollar figure is + * backward-looking so the operator can check it against a bill already + * paid. Ceilings are labeled as ceilings in estimateBasis. + * - THREE-STATE HONESTY: 'actionable' (evidence found), 'clean' (ran, + * found nothing), 'not_measured' (data doesn't exist yet on this + * install). An un-run scan is not an all-clear. + * - MECHANISM badges: 'auto' (Lynkr could act), 'snippet' (client-side + * fix the operator applies), 'info' (diagnosis only). + * + * Analyzer contract: analyze({db, since, until}) -> finding + * {id, title, mechanism, state, framing: 'usd'|'tokens', + * pastOverspendUsd|null, pastOverspendTokens|null, stats: [{label,value}], + * evidence: {columns:[], rows:[][]}, fix, caveat, estimateBasis} + * + * The analyzers' overspend bases are disjoint by construction (dead tool + * schemas / harness side traffic / uncached input / tier price deltas), so + * the total is a plain sum — revisit if an analyzer is added whose basis + * overlaps an existing one. + * + * @module dashboard/recommendations + */ + +const logger = require('../../logger'); +const telemetry = require('../../routing/telemetry'); + +const ANALYZERS = [ + require('./deadweight'), + require('./side-requests'), + require('./cache-weakspots'), + require('./downsize'), +]; + +const DEFAULT_WINDOW_MS = 7 * 24 * 60 * 60 * 1000; +const RESULT_TTL_MS = 45 * 1000; + +let _cached = null; + +/** + * Run all analyzers. Cached for 45s — findings shift slowly and the + * dashboard polls every 30s. + * + * @param {Object} [opts] {windowMs, force} + */ +function run(opts = {}) { + const windowMs = opts.windowMs ?? DEFAULT_WINDOW_MS; + if (!opts.force && _cached && Date.now() - _cached.computedAt < RESULT_TTL_MS + && _cached.windowMs === windowMs) { + return _cached; + } + + const db = telemetry.getDb(); + const until = Date.now(); + const since = until - windowMs; + + const findings = []; + for (const analyzer of ANALYZERS) { + try { + const finding = db + ? analyzer.analyze({ db, since, until }) + : { id: analyzer.id, title: analyzer.title, state: 'not_measured', reason: 'telemetry unavailable' }; + if (finding) findings.push(finding); + } catch (err) { + logger.debug({ analyzer: analyzer.id, err: err.message }, '[Recommendations] analyzer failed'); + findings.push({ id: analyzer.id, title: analyzer.title, state: 'not_measured', reason: err.message }); + } + } + + // Rank by dollars, biggest first; token-framed findings after usd ones. + findings.sort((a, b) => (b.pastOverspendUsd ?? -1) - (a.pastOverspendUsd ?? -1)); + + let spendUsd = 0; + try { + if (db) { + spendUsd = db + .prepare('SELECT SUM(COALESCE(cost_usd,0)) s FROM routing_telemetry WHERE timestamp > ?') + .get(since)?.s ?? 0; + } + } catch { /* spend share stays 0 */ } + + const totalRecoverableUsd = findings + .filter((f) => f.state === 'actionable' && typeof f.pastOverspendUsd === 'number') + .reduce((acc, f) => acc + f.pastOverspendUsd, 0); + + _cached = { + computedAt: Date.now(), + windowMs, + spendUsd, + totalRecoverableUsd, + findings, + }; + return _cached; +} + +/** Test helper — drop the memoized result. */ +function _clearCacheForTests() { + _cached = null; +} + +module.exports = { run, ANALYZERS, _clearCacheForTests }; diff --git a/src/dashboard/recommendations/side-requests.js b/src/dashboard/recommendations/side-requests.js new file mode 100644 index 0000000..18b6d70 --- /dev/null +++ b/src/dashboard/recommendations/side-requests.js @@ -0,0 +1,68 @@ +/** + * side-requests — harness side traffic served above the SIMPLE tier. + * + * Coding harnesses replay the conversation for their own bookkeeping + * (title generation, topic detection). Those calls embed the user's text, + * so they score like the user's request and ride the same tier — observed + * live: a title request served by a frontier COMPLEX model. The whole + * category needs a tiny model. + */ + +const SIDE_PATTERNS = [ + "Generate a title%", + "%Analyze if this message indicates a new conversation topic%", + "Please write a %commit message%", + "Summarize this conversation%", +]; + +module.exports = { + id: 'side-requests', + title: 'Harness side traffic on expensive tiers', + + analyze({ db, since }) { + const where = SIDE_PATTERNS.map(() => 'request_text LIKE ?').join(' OR '); + const rows = db + .prepare( + `SELECT tier, provider, model, COUNT(*) n, SUM(COALESCE(cost_usd,0)) cost + FROM routing_telemetry + WHERE timestamp > ? AND COALESCE(tool_count,0) = 0 AND (${where}) + GROUP BY tier, provider, model + ORDER BY cost DESC` + ) + .all(since, ...SIDE_PATTERNS); + + const above = rows.filter((r) => r.tier && r.tier !== 'SIMPLE'); + const total = rows.reduce((a, r) => a + r.n, 0); + if (!above.length) { + return { + id: this.id, title: this.title, mechanism: 'auto', state: total > 0 ? 'clean' : 'clean', + framing: 'usd', pastOverspendUsd: 0, pastOverspendTokens: null, + stats: [{ label: 'side requests seen', value: total }], + evidence: { columns: [], rows: [] }, fix: null, caveat: null, + estimateBasis: 'Tool-less requests matching known harness bookkeeping prompts (title/topic/commit/summary).', + }; + } + + const usd = above.reduce((a, r) => a + r.cost, 0); + return { + id: this.id, + title: this.title, + mechanism: 'auto', + state: 'actionable', + framing: 'usd', + pastOverspendUsd: usd, + pastOverspendTokens: null, + stats: [ + { label: 'side requests above SIMPLE', value: above.reduce((a, r) => a + r.n, 0) }, + { label: 'total side requests', value: total }, + ], + evidence: { + columns: ['tier', 'provider', 'model', 'requests', 'spend $'], + rows: above.map((r) => [r.tier, r.provider, r.model, r.n, r.cost]), + }, + fix: 'Route harness bookkeeping (title generation, topic detection, commit messages) to the SIMPLE tier unconditionally — a pattern guard ahead of intent scoring. Lynkr can enforce this; the patterns above are the trigger list.', + caveat: 'Ceiling figure: it counts the full spend of these calls, not the delta vs a SIMPLE-tier serve (which would be near zero for local SIMPLE tiers).', + estimateBasis: 'Sum of cost_usd for tool-less requests matching harness prompt patterns, served on any tier above SIMPLE, this window.', + }; + }, +}; diff --git a/src/dashboard/recommendations/wilson.js b/src/dashboard/recommendations/wilson.js new file mode 100644 index 0000000..c3276da --- /dev/null +++ b/src/dashboard/recommendations/wilson.js @@ -0,0 +1,24 @@ +/** + * Wilson score interval — lower bound on a binomial proportion. + * + * Used by the downsize analyzer: "the lower tier succeeded k of n times" + * is only evidence when the LOWER BOUND of the confidence interval clears + * the bar. A raw average (k/n >= 0.7) treats 21/30 the same as 700/1000; + * Wilson does not — small samples get wide intervals and stay unproven. + * + * @param {number} successes + * @param {number} n + * @param {number} [z=1.96] - 95% confidence + * @returns {number} lower bound in [0,1]; 0 when n === 0 + */ +function wilsonLowerBound(successes, n, z = 1.96) { + if (!Number.isFinite(n) || n <= 0) return 0; + const p = Math.min(1, Math.max(0, successes / n)); + const z2 = z * z; + const denom = 1 + z2 / n; + const centre = p + z2 / (2 * n); + const margin = z * Math.sqrt((p * (1 - p) + z2 / (4 * n)) / n); + return Math.max(0, (centre - margin) / denom); +} + +module.exports = { wilsonLowerBound }; diff --git a/src/dashboard/router.js b/src/dashboard/router.js index b3cb328..caf91da 100644 --- a/src/dashboard/router.js +++ b/src/dashboard/router.js @@ -18,5 +18,10 @@ router.get('/api/overview', api.overview); router.get('/api/usage', api.usage); router.get('/api/routing', api.routing); router.get('/api/logs', api.logs); +// Lens (feature/lens-dashboard) +router.get('/api/recommendations', api.recommendations); +router.get('/api/sessions/:id', api.sessionDetail); +router.get('/api/analytics', api.analytics); +router.get('/api/statusline', api.statusline); module.exports = router; diff --git a/src/routing/telemetry.js b/src/routing/telemetry.js index f9f3f7f..81caca2 100644 --- a/src/routing/telemetry.js +++ b/src/routing/telemetry.js @@ -170,6 +170,11 @@ function init() { // projectedSwitchCostUsd, projectedStaySavingsUsd, // expectedRemainingTurns} as JSON. ["cache_decision", "TEXT"], + // Lens dashboard — per-request cache counters from the provider's + // usage payload. NULL on rows recorded before capture began (or by + // providers that report none) — "not measured" is distinct from 0. + ["cache_read_tokens", "INTEGER"], + ["cache_creation_tokens", "INTEGER"], ]; for (const [col, type] of additiveCols) { if (!existingCols.has(col)) { @@ -232,7 +237,7 @@ function record(data) { retry_count, circuit_breaker_state, quality_score, tokens_per_second, cost_efficiency, request_text, response_text, base_tier, escalation_source, propensity, candidates, pinned, switch_reason, - cache_decision + cache_decision, cache_read_tokens, cache_creation_tokens ) VALUES ( @request_id, @session_id, @timestamp, @complexity_score, @tier, @agentic_type, @tool_count, @input_tokens, @message_count, @request_type, @@ -241,7 +246,7 @@ function record(data) { @retry_count, @circuit_breaker_state, @quality_score, @tokens_per_second, @cost_efficiency, @request_text, @response_text, @base_tier, @escalation_source, @propensity, @candidates, @pinned, @switch_reason, - @cache_decision + @cache_decision, @cache_read_tokens, @cache_creation_tokens )` ); if (!insert) return; @@ -292,6 +297,8 @@ function record(data) { : (typeof data.cache_decision === "string" ? data.cache_decision : JSON.stringify(data.cache_decision)), + cache_read_tokens: data.cache_read_tokens ?? null, + cache_creation_tokens: data.cache_creation_tokens ?? null, }); } catch (err) { logger.debug({ err: err.message }, "Telemetry record failed"); @@ -926,6 +933,200 @@ function getExpectedRemainingTurns(currentTurns = 0) { } } +// --------------------------------------------------------------------------- +// Lens dashboard queries +// --------------------------------------------------------------------------- + +/** + * MEASURED cache savings — backward-looking, auditable against the bill: + * for rows that carried cache counters, what did cache reads actually save + * vs. paying full input price? Distinct from the projected figures in + * getCacheEconomics. Also reports the hit ratio per (provider, model). + * + * @param {Object} [opts] {since} + * @returns {{measuredSavedUsd:number, rowsMeasured:number, perModel:Array}|null} + */ +function getMeasuredCacheSavings(opts = {}) { + if (!init()) return null; + const since = opts.since ?? Date.now() - 7 * 24 * 60 * 60 * 1000; + try { + const rows = db + .prepare( + `SELECT provider, model, COUNT(*) n, + SUM(COALESCE(cache_read_tokens,0)) rd, + SUM(COALESCE(cache_creation_tokens,0)) cr, + SUM(COALESCE(input_tokens,0)) inp + FROM routing_telemetry + WHERE timestamp > ? AND cache_read_tokens IS NOT NULL + GROUP BY provider, model` + ) + .all(since); + const { resolveCacheEconomics } = require("./cache-economics"); + let saved = 0; + let measured = 0; + const perModel = []; + for (const r of rows) { + measured += r.n; + const econ = resolveCacheEconomics(r.provider, r.model); + const savedUsd = econ.unknownPricing + ? null + : (r.rd * Math.max(0, econ.inputPerM - econ.cacheReadPerM)) / 1_000_000; + if (savedUsd != null) saved += savedUsd; + const total = r.rd + r.cr + r.inp; + perModel.push({ + provider: r.provider, + model: r.model, + requests: r.n, + cacheReadTokens: r.rd, + hitRatio: total > 0 ? r.rd / total : null, + savedUsd, + }); + } + perModel.sort((a, b) => (a.hitRatio ?? 1) - (b.hitRatio ?? 1)); + return { measuredSavedUsd: saved, rowsMeasured: measured, perModel }; + } catch (err) { + logger.debug({ err: err.message }, "Telemetry getMeasuredCacheSavings failed"); + return null; + } +} + +/** + * Session drill-down: per-model mix, context-growth series, tool totals. + * @param {string} sessionId + */ +function getSessionDetail(sessionId) { + if (!sessionId || !init()) return null; + try { + const rows = db + .prepare( + `SELECT timestamp, provider, model, tier, routing_method, input_tokens, + output_tokens, cost_usd, latency_ms, tool_count, tool_calls_made, + cache_read_tokens, cache_creation_tokens, status_code, error_type, + pinned, switch_reason, message_count + FROM routing_telemetry WHERE session_id = ? ORDER BY timestamp ASC LIMIT 2000` + ) + .all(sessionId); + if (!rows.length) return null; + + const perModel = new Map(); + let toolCalls = 0; + let cost = 0; + for (const r of rows) { + const key = `${r.provider}:${r.model}`; + const m = perModel.get(key) ?? { + provider: r.provider, model: r.model, requests: 0, + inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, costUsd: 0, + }; + m.requests++; + m.inputTokens += r.input_tokens ?? 0; + m.outputTokens += r.output_tokens ?? 0; + m.cacheReadTokens += r.cache_read_tokens ?? 0; + m.costUsd += r.cost_usd ?? 0; + perModel.set(key, m); + toolCalls += r.tool_calls_made ?? 0; + cost += r.cost_usd ?? 0; + } + + return { + sessionId, + requests: rows.length, + totalCostUsd: cost, + toolCalls, + firstSeen: rows[0].timestamp, + lastSeen: rows[rows.length - 1].timestamp, + models: [...perModel.values()].sort((a, b) => b.costUsd - a.costUsd), + // Input (context) tokens per call over the session — the distiller + // cliff and frozen-block plateau are visible here. + contextSeries: rows.map((r) => ({ + t: r.timestamp, + inputTokens: r.input_tokens ?? 0, + cacheReadTokens: r.cache_read_tokens ?? null, + tier: r.tier, + model: r.model, + })), + rows, + }; + } catch (err) { + logger.debug({ err: err.message }, "Telemetry getSessionDetail failed"); + return null; + } +} + +// Pivot explorer — every metric/dimension is whitelisted; params never reach +// SQL as strings. +const ANALYTICS_METRICS = { + spend: "SUM(COALESCE(cost_usd,0))", + tokens: "SUM(COALESCE(input_tokens,0)+COALESCE(output_tokens,0))", + requests: "COUNT(*)", + sessions: "COUNT(DISTINCT session_id)", +}; +const ANALYTICS_DIMS = { + provider: "provider", + model: "model", + tier: "tier", + request_type: "request_type", + routing_method: "routing_method", + switch_reason: "switch_reason", + day: "date(timestamp/1000, 'unixepoch')", +}; + +/** + * Generic Metric × Dimension (× Stack) pivot with KPI row and + * period-over-period deltas (suppressed when the prior window is thin). + * + * @param {Object} [opts] {metric, by, stack, since, until} + */ +function getAnalytics(opts = {}) { + if (!init()) return null; + const metricExpr = ANALYTICS_METRICS[opts.metric] ?? ANALYTICS_METRICS.spend; + const metric = ANALYTICS_METRICS[opts.metric] ? opts.metric : "spend"; + const byExpr = ANALYTICS_DIMS[opts.by] ?? ANALYTICS_DIMS.provider; + const by = ANALYTICS_DIMS[opts.by] ? opts.by : "provider"; + const stackExpr = opts.stack && ANALYTICS_DIMS[opts.stack] ? ANALYTICS_DIMS[opts.stack] : null; + const until = opts.until ?? Date.now(); + const since = opts.since ?? until - 7 * 24 * 60 * 60 * 1000; + + try { + const groupCols = stackExpr ? `${byExpr} AS dim, ${stackExpr} AS stack` : `${byExpr} AS dim`; + const groupBy = stackExpr ? "GROUP BY dim, stack" : "GROUP BY dim"; + const rows = db + .prepare( + `SELECT ${groupCols}, ${metricExpr} AS value + FROM routing_telemetry + WHERE timestamp BETWEEN ? AND ? + ${groupBy} ORDER BY value DESC LIMIT 500` + ) + .all(since, until); + + const kpiRow = (lo, hi) => + db + .prepare( + `SELECT SUM(COALESCE(cost_usd,0)) spend, + SUM(COALESCE(input_tokens,0)+COALESCE(output_tokens,0)) tokens, + COUNT(*) requests, + COUNT(DISTINCT session_id) sessions + FROM routing_telemetry WHERE timestamp BETWEEN ? AND ?` + ) + .get(lo, hi); + const kpis = kpiRow(since, until); + const prev = kpiRow(since - (until - since), since); + // Thin prior window → deltas are noise, suppress rather than mislead. + const kpiDeltas = (prev?.requests ?? 0) >= 10 + ? { + spend: kpis.spend - (prev.spend ?? 0), + tokens: kpis.tokens - (prev.tokens ?? 0), + requests: kpis.requests - prev.requests, + sessions: kpis.sessions - (prev.sessions ?? 0), + } + : null; + + return { metric, by, stack: stackExpr ? opts.stack : null, since, until, rows, kpis, kpiDeltas }; + } catch (err) { + logger.debug({ err: err.message }, "Telemetry getAnalytics failed"); + return null; + } +} + module.exports = { record, query, @@ -936,6 +1137,9 @@ module.exports = { getQualityByTierAndType, getExpectedRemainingTurns, getCacheEconomics, + getMeasuredCacheSavings, + getSessionDetail, + getAnalytics, recordSavings, getSavingsSummary, cleanup, diff --git a/test/lens-recommendations.test.js b/test/lens-recommendations.test.js new file mode 100644 index 0000000..cc3a6ab --- /dev/null +++ b/test/lens-recommendations.test.js @@ -0,0 +1,268 @@ +const assert = require("assert"); +const { describe, it, beforeEach, after } = require("node:test"); +const fs = require("fs"); +const os = require("os"); +const path = require("path"); + +// Isolate the shared telemetry SQLite before anything touches it. +const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "lynkr-lens-")); +const telemetry = require("../src/routing/telemetry"); +telemetry._setDbPathForTests(path.join(tmpDir, "telemetry.db")); + +const { wilsonLowerBound } = require("../src/dashboard/recommendations/wilson"); +const engine = require("../src/dashboard/recommendations"); + +after(() => { + try { + fs.rmSync(tmpDir, { recursive: true, force: true }); + } catch { /* best-effort */ } +}); + +function insertRow(over = {}) { + const db = telemetry.getDb(); + db.prepare( + `INSERT INTO routing_telemetry ( + request_id, session_id, timestamp, provider, model, tier, request_type, + tool_count, tool_calls_made, input_tokens, output_tokens, cost_usd, + quality_score, error_type, request_text, cache_read_tokens, + cache_creation_tokens, pinned + ) VALUES ( + @request_id, @session_id, @timestamp, @provider, @model, @tier, @request_type, + @tool_count, @tool_calls_made, @input_tokens, @output_tokens, @cost_usd, + @quality_score, @error_type, @request_text, @cache_read_tokens, + @cache_creation_tokens, @pinned + )` + ).run({ + request_id: over.request_id ?? `r-${Math.random()}`, + session_id: over.session_id ?? null, + timestamp: over.timestamp ?? Date.now(), + provider: over.provider ?? "databricks", + model: over.model ?? "databricks-claude-haiku-4-5", + tier: over.tier ?? "MEDIUM", + request_type: over.request_type ?? null, + tool_count: over.tool_count ?? 0, + tool_calls_made: over.tool_calls_made ?? 0, + input_tokens: over.input_tokens ?? 1000, + output_tokens: over.output_tokens ?? 200, + cost_usd: over.cost_usd ?? 0.01, + quality_score: over.quality_score ?? null, + error_type: over.error_type ?? null, + request_text: over.request_text ?? "normal user request", + cache_read_tokens: over.cache_read_tokens ?? null, + cache_creation_tokens: over.cache_creation_tokens ?? null, + pinned: over.pinned ?? 0, + }); +} + +function clearRows() { + telemetry.getDb().prepare("DELETE FROM routing_telemetry").run(); + engine._clearCacheForTests(); +} + +describe("wilson lower bound", () => { + it("is conservative on small samples, tighter on large ones", () => { + const small = wilsonLowerBound(21, 30); // 70% raw + const large = wilsonLowerBound(700, 1000); // 70% raw + assert.ok(small < 0.55, `small-sample bound should be well below 0.7, got ${small}`); + assert.ok(large > 0.67, `large-sample bound should approach 0.7, got ${large}`); + assert.strictEqual(wilsonLowerBound(0, 0), 0); + assert.ok(wilsonLowerBound(10, 10) < 1); + }); +}); + +describe("recommendations engine", () => { + beforeEach(clearRows); + + it("deadweight flags all-tools-no-calls sessions and prices the rent", () => { + for (let i = 0; i < 5; i++) { + insertRow({ session_id: "dead-1", tool_count: 12, tool_calls_made: 0 }); + } + // Control: session that DID call tools must not appear. + for (let i = 0; i < 5; i++) { + insertRow({ session_id: "alive-1", tool_count: 12, tool_calls_made: 3 }); + } + const out = engine.run({ force: true }); + const dw = out.findings.find((f) => f.id === "deadweight"); + assert.strictEqual(dw.state, "actionable"); + assert.strictEqual(dw.evidence.rows.length, 1); + assert.strictEqual(dw.evidence.rows[0][0], "dead-1"); + // 12 tools × 150 tok × 5 reqs = 9000 est. tokens + assert.strictEqual(dw.evidence.rows[0][3], 9000); + assert.ok(dw.pastOverspendUsd > 0); + }); + + it("side-requests flags harness traffic above SIMPLE only", () => { + insertRow({ request_text: "Generate a title for this conversation: how do I fix X", tier: "COMPLEX", cost_usd: 0.05 }); + insertRow({ request_text: "Generate a title for this conversation: hello", tier: "SIMPLE", cost_usd: 0.0001 }); + const out = engine.run({ force: true }); + const sr = out.findings.find((f) => f.id === "side-requests"); + assert.strictEqual(sr.state, "actionable"); + assert.strictEqual(sr.evidence.rows.length, 1); + assert.strictEqual(sr.evidence.rows[0][0], "COMPLEX"); + assert.ok(Math.abs(sr.pastOverspendUsd - 0.05) < 1e-9); + }); + + it("cache-weakspots distinguishes not_measured from clean from weak", () => { + let out = engine.run({ force: true }); + assert.strictEqual(out.findings.find((f) => f.id === "cache-weakspots").state, "not_measured"); + + // Healthy: high hit ratio. + for (let i = 0; i < 6; i++) { + insertRow({ cache_read_tokens: 9000, cache_creation_tokens: 500, input_tokens: 500 }); + } + out = engine.run({ force: true }); + assert.strictEqual(out.findings.find((f) => f.id === "cache-weakspots").state, "clean"); + + // Weak: model that never hits. + for (let i = 0; i < 6; i++) { + insertRow({ model: "databricks-claude-opus-4-6", cache_read_tokens: 0, cache_creation_tokens: 100, input_tokens: 10000 }); + } + out = engine.run({ force: true }); + const cw = out.findings.find((f) => f.id === "cache-weakspots"); + assert.strictEqual(cw.state, "actionable"); + assert.strictEqual(cw.evidence.rows.length, 1); + assert.ok(cw.pastOverspendUsd > 0); + }); + + it("downsize requires the Wilson bound to clear, not the raw average", () => { + // Upper-tier spend on request_type 'general'. + for (let i = 0; i < 10; i++) { + insertRow({ tier: "COMPLEX", request_type: "general", cost_usd: 0.1 }); + } + // Lower tier: 21/30 successes — raw 70% but Wilson-lower ~0.52 → unproven. + for (let i = 0; i < 30; i++) { + insertRow({ tier: "MEDIUM", request_type: "general", quality_score: i < 21 ? 90 : 40, cost_usd: 0.01 }); + } + let out = engine.run({ force: true }); + let dz = out.findings.find((f) => f.id === "downsize"); + assert.strictEqual(dz.state, "clean", "21/30 must NOT prove the lower tier"); + + // Now overwhelming evidence: 170 more successes. + for (let i = 0; i < 170; i++) { + insertRow({ tier: "MEDIUM", request_type: "general", quality_score: 90, cost_usd: 0.01 }); + } + out = engine.run({ force: true }); + dz = out.findings.find((f) => f.id === "downsize"); + assert.strictEqual(dz.state, "actionable"); + assert.strictEqual(dz.evidence.rows[0][0], "general"); + assert.ok(dz.pastOverspendUsd > 0); + }); + + it("totals sum only actionable usd findings and reports spend share basis", () => { + for (let i = 0; i < 5; i++) { + insertRow({ session_id: "dead-2", tool_count: 10, tool_calls_made: 0, cost_usd: 0.02 }); + } + const out = engine.run({ force: true }); + assert.ok(out.totalRecoverableUsd > 0); + assert.ok(out.spendUsd >= 0.1 - 1e-9); + assert.ok(out.computedAt > 0); + // Ranked by dollars descending among usd findings. + const usd = out.findings.filter((f) => typeof f.pastOverspendUsd === "number"); + for (let i = 1; i < usd.length; i++) { + assert.ok(usd[i - 1].pastOverspendUsd >= usd[i].pastOverspendUsd); + } + }); + + it("memoizes results for 45s and force bypasses", () => { + const a = engine.run({ force: true }); + insertRow({ session_id: "dead-3", tool_count: 10, tool_calls_made: 0 }); + const b = engine.run(); + assert.strictEqual(a.computedAt, b.computedAt, "cached result served"); + const c = engine.run({ force: true }); + assert.notStrictEqual(a.computedAt <= c.computedAt && a === c, true); + }); +}); + +describe("telemetry lens queries", () => { + beforeEach(clearRows); + + it("getMeasuredCacheSavings prices actual reads and sorts weakest first", () => { + for (let i = 0; i < 3; i++) { + insertRow({ model: "databricks-claude-haiku-4-5", cache_read_tokens: 100000, cache_creation_tokens: 0, input_tokens: 1000 }); + } + const m = telemetry.getMeasuredCacheSavings(); + assert.strictEqual(m.rowsMeasured, 3); + // haiku: input $1/M, cacheRead ~$0.1/M → 300k reads save ≈$0.27 (registry + // data may differ from the fallback multiplier at the 4th decimal) + assert.ok(Math.abs(m.measuredSavedUsd - 0.27) < 5e-3, `got ${m.measuredSavedUsd}`); + assert.ok(m.perModel[0].hitRatio > 0.9); + }); + + it("getSessionDetail aggregates per-model mix and context series", () => { + const t0 = Date.now() - 10000; + for (let i = 0; i < 4; i++) { + insertRow({ session_id: "sess-detail", timestamp: t0 + i * 1000, input_tokens: 1000 * (i + 1), cost_usd: 0.01 }); + } + const d = telemetry.getSessionDetail("sess-detail"); + assert.strictEqual(d.requests, 4); + assert.strictEqual(d.models.length, 1); + assert.strictEqual(d.contextSeries.length, 4); + assert.strictEqual(d.contextSeries[3].inputTokens, 4000); + assert.strictEqual(telemetry.getSessionDetail("nope"), null); + }); + + it("getAnalytics pivots with whitelisted dims and suppresses thin deltas", () => { + for (let i = 0; i < 12; i++) { + insertRow({ provider: i % 2 ? "ollama" : "databricks", tier: i % 2 ? "SIMPLE" : "COMPLEX", cost_usd: 0.01 }); + } + const a = telemetry.getAnalytics({ metric: "requests", by: "provider" }); + assert.strictEqual(a.metric, "requests"); + assert.strictEqual(a.rows.length, 2); + assert.strictEqual(a.kpis.requests, 12); + assert.strictEqual(a.kpiDeltas, null, "prior window is empty → deltas suppressed"); + + // Injection-shaped params fall back to whitelisted defaults. + const b = telemetry.getAnalytics({ metric: "spend; DROP TABLE", by: "1=1" }); + assert.strictEqual(b.metric, "spend"); + assert.strictEqual(b.by, "provider"); + + const c = telemetry.getAnalytics({ metric: "spend", by: "tier", stack: "provider" }); + assert.ok(c.rows.every((r) => "stack" in r)); + }); +}); + +describe("lens api handlers", () => { + beforeEach(clearRows); + + function call(handler, { params = {}, query = {} } = {}) { + let status = 200; let body = null; + const res = { + status(s) { status = s; return this; }, + json(b) { body = b; }, + }; + handler({ params, query }, res); + return { status, body }; + } + + it("recommendations endpoint returns the engine artifact", () => { + const api = require("../src/dashboard/api"); + const { status, body } = call(api.recommendations, { query: { force: "true" } }); + assert.strictEqual(status, 200); + assert.ok(Array.isArray(body.findings)); + assert.strictEqual(body.findings.length, 4); + }); + + it("sessionDetail 404s unknown sessions", () => { + const api = require("../src/dashboard/api"); + assert.strictEqual(call(api.sessionDetail, { params: { id: "missing" } }).status, 404); + insertRow({ session_id: "s-api" }); + assert.strictEqual(call(api.sessionDetail, { params: { id: "s-api" } }).status, 200); + }); + + it("statusline returns last-request snapshot and cache share", () => { + insertRow({ tier: "MEDIUM", provider: "ollama", model: "m3", cache_read_tokens: 900, cache_creation_tokens: 0, input_tokens: 100 }); + const api = require("../src/dashboard/api"); + const { status, body } = call(api.statusline); + assert.strictEqual(status, 200); + assert.strictEqual(body.last.provider, "ollama"); + assert.strictEqual(body.cacheReadPct, 90); + }); + + it("analytics endpoint clamps days and rejects nothing (whitelist fallback)", () => { + insertRow({}); + const api = require("../src/dashboard/api"); + const { status, body } = call(api.analytics, { query: { metric: "tokens", by: "model", days: "9999" } }); + assert.strictEqual(status, 200); + assert.strictEqual(body.metric, "tokens"); + }); +}); diff --git a/test/memory/tencentdb-launcher.test.js b/test/memory/tencentdb-launcher.test.js index 6421e7b..73e3312 100644 --- a/test/memory/tencentdb-launcher.test.js +++ b/test/memory/tencentdb-launcher.test.js @@ -36,10 +36,10 @@ describe("TencentDB Memory Launcher", () => { }); describe("config defaults", () => { - it("is disabled by default", () => { + it("is enabled by default", () => { delete process.env.TENCENTDB_MEMORY_ENABLED; const config = require("../../src/config"); - assert.strictEqual(config.tencentdbMemory.enabled, false); + assert.strictEqual(config.tencentdbMemory.enabled, true); }); it("enables via TENCENTDB_MEMORY_ENABLED=true", () => { @@ -69,7 +69,7 @@ describe("TencentDB Memory Launcher", () => { describe("ensureRunning() gating", () => { it("skips when disabled", async () => { - delete process.env.TENCENTDB_MEMORY_ENABLED; + process.env.TENCENTDB_MEMORY_ENABLED = "false"; const launcher = require("../../src/memory/tencentdb-launcher"); const result = await launcher.ensureRunning(); assert.strictEqual(result.started, false);