Fix hybrid-mode scoring: log a cosine relevance score, not an RRF rank score

docs/analytics.html

@@ -477,7 +477,15 @@ <h2 id="topQueriesTitle" style="margin-bottom: 12px">Top Queries</h2>
           <th>Tool</th>
           <th>Count</th>
           <th>Avg Results</th>
-          <th>Avg Score</th>
+          <!-- "Avg Cosine", not "Avg Score": the column shows the mean
+               best-match cosine similarity (0-1). It deliberately reads "—"
+               for keyword-only traffic and for rows logged before the scale
+               was recorded, rather than mixing a rank score into the mean. -->
+          <th
+            title="Mean best-match cosine similarity (0-1) over this query's events. Blank when no event recorded a cosine score (keyword-only matches, browse calls, or rows logged before the score scale was tracked)."
+          >
+            Avg Cosine
+          </th>
         </tr>
       </thead>
       <tbody></tbody>

src/__tests__/analytics-observability.test.ts

@@ -9,6 +9,7 @@ import {
   ALL_TIME_DAYS,
   ROLLING_WINDOW_CAP_DAYS,
   BROWSE_QUERY_TEXT,
+  COSINE_SCORE_KIND,
 } from "../db/analytics.js";
 import { generatePostSchemaMigration } from "../db/schema.js";
 
@@ -72,14 +73,21 @@ async function seed(db: PGlite, count: number, opts: SeedOpts = {}) {
   for (let i = 0; i < count; i++) {
     await db.query(
       `INSERT INTO query_log
-        (tool_name, query_text, result_count, top_score, latency_ms,
+        (tool_name, query_text, result_count, top_score, score_kind, latency_ms,
          source_name, session_id, request_source, created_at)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
       [
         "search-docs",
         opts.query_text ?? "q",
         opts.result_count ?? 5,
         opts.top_score === undefined ? 0.9 : opts.top_score,
+        // Mirror logQuery's write-boundary rule: a present score declares its
+        // scale, an absent one declares nothing. Score-based readers require
+        // the tag, so seeding without it would silently test the "unknown
+        // scale, excluded" path instead of the intended one.
+        (opts.top_score === undefined ? 0.9 : opts.top_score) == null
+          ? null
+          : COSINE_SCORE_KIND,
         42,
         "docs",
         "sess-1",
@@ -298,14 +306,17 @@ async function seedAt(
 ): Promise<void> {
   await db.query(
     `INSERT INTO query_log
-        (tool_name, query_text, result_count, top_score, latency_ms,
+        (tool_name, query_text, result_count, top_score, score_kind, latency_ms,
          source_name, session_id, request_source, created_at)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)`,
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)`,
     [
       "search-docs",
       opts.query_text ?? "q",
       opts.result_count ?? 5,
       opts.top_score === undefined ? 0.9 : opts.top_score,
+      (opts.top_score === undefined ? 0.9 : opts.top_score) == null
+        ? null
+        : COSINE_SCORE_KIND,
       42,
       "docs",
       "sess-1",

src/__tests__/analytics.test.ts

@@ -18,6 +18,7 @@ import {
   REDACTED_QUERY_TEXT,
   P95_LATENCY_ROW_CAP,
   LOW_CONFIDENCE_SCORE_THRESHOLD,
+  COSINE_SCORE_KIND,
   normalizeRequestSource,
   DEFAULT_REQUEST_SOURCE,
   REQUEST_SOURCE_VALUES,
@@ -63,6 +64,9 @@ describe("logQuery", () => {
       "how to install",
       5,
       0.92,
+      // score_kind: derived from top_score at the write boundary, so a present
+      // score is always tagged with the scale it lives on.
+      COSINE_SCORE_KIND,
       42,
       "docs",
       "sess-123",
@@ -93,6 +97,7 @@ describe("logQuery", () => {
       REDACTED_QUERY_TEXT,
       baseEntry.result_count,
       baseEntry.top_score,
+      COSINE_SCORE_KIND,
       baseEntry.latency_ms,
       baseEntry.source_name,
       baseEntry.session_id,
@@ -119,8 +124,11 @@ describe("logQuery", () => {
 
     const [, params] = mockQuery.mock.calls[0];
     expect(params[3]).toBeNull(); // top_score
-    expect(params[5]).toBeNull(); // source_name
-    expect(params[6]).toBeNull(); // session_id
+    // A NULL top_score must carry a NULL score_kind — an untagged absence, not
+    // a score on an unnamed scale.
+    expect(params[4]).toBeNull(); // score_kind
+    expect(params[6]).toBeNull(); // source_name
+    expect(params[7]).toBeNull(); // session_id
   });
 
   it("persists the session_id passed on the entry (no longer hardcoded null)", async () => {
@@ -131,7 +139,7 @@ describe("logQuery", () => {
     await logQuery({ ...baseEntry, session_id: "live-session-42" });
 
     const [, params] = mockQuery.mock.calls[0];
-    expect(params[6]).toBe("live-session-42");
+    expect(params[7]).toBe("live-session-42");
   });
 
   it("coerces an unknown request_source to the default ('user')", async () => {
@@ -141,7 +149,7 @@ describe("logQuery", () => {
     await logQuery({ ...baseEntry, request_source: "bogus-origin" });
 
     const [, params] = mockQuery.mock.calls[0];
-    expect(params[7]).toBe("user");
+    expect(params[8]).toBe("user");
   });
 
   it("coerces an absent request_source to the default ('user')", async () => {
@@ -151,20 +159,20 @@ describe("logQuery", () => {
     await logQuery(noSource);
 
     const [, params] = mockQuery.mock.calls[0];
-    expect(params[7]).toBe("user");
+    expect(params[8]).toBe("user");
   });
 
   it("persists a synthetic request_source verbatim", async () => {
     mockQuery.mockResolvedValueOnce({ rows: [] });
     await logQuery({ ...baseEntry, request_source: "synthetic" });
 
     const [, params] = mockQuery.mock.calls[0];
-    expect(params[7]).toBe("synthetic");
+    expect(params[8]).toBe("synthetic");
   });
 
   // v1.15.2 attribution columns: client_ip, user_agent, blocked, block_reason.
-  // Positional indices on params are 8, 9, 10, 11 respectively (after the
-  // existing 8 fields tool_name..request_source).
+  // Positional indices on params are 9, 10, 11, 12 respectively (after the
+  // nine fields tool_name..request_source, which now include score_kind).
 
   it("persists client_ip and user_agent verbatim when provided", async () => {
     mockQuery.mockResolvedValueOnce({ rows: [] });
@@ -175,8 +183,8 @@ describe("logQuery", () => {
     });
 
     const [, params] = mockQuery.mock.calls[0];
-    expect(params[8]).toBe("203.0.113.10");
-    expect(params[9]).toBe("Claude-User/1.0");
+    expect(params[9]).toBe("203.0.113.10");
+    expect(params[10]).toBe("Claude-User/1.0");
   });
 
   it("truncates a pathological user_agent to USER_AGENT_MAX_LEN chars", async () => {
@@ -188,7 +196,7 @@ describe("logQuery", () => {
     await logQuery({ ...baseEntry, user_agent: huge });
 
     const [, params] = mockQuery.mock.calls[0];
-    expect((params[9] as string).length).toBe(256);
+    expect((params[10] as string).length).toBe(256);
   });
 
   it("persists blocked=true with a block_reason", async () => {
@@ -200,8 +208,8 @@ describe("logQuery", () => {
     });
 
     const [, params] = mockQuery.mock.calls[0];
-    expect(params[10]).toBe(true);
-    expect(params[11]).toBe("pattern:movie-box-office");
+    expect(params[11]).toBe(true);
+    expect(params[12]).toBe("pattern:movie-box-office");
   });
 
   it("defaults blocked to false and block_reason to null when absent", async () => {
@@ -211,8 +219,8 @@ describe("logQuery", () => {
     await logQuery(baseEntry);
 
     const [, params] = mockQuery.mock.calls[0];
-    expect(params[10]).toBe(false);
-    expect(params[11]).toBeNull();
+    expect(params[11]).toBe(false);
+    expect(params[12]).toBeNull();
   });
 });
 
@@ -1833,6 +1841,31 @@ describe("getAnalyticsSummary low-confidence metric", () => {
     expect(params).toContain(LOW_CONFIDENCE_SCORE_THRESHOLD);
   });
 
+  it("gates the low-confidence FILTER on score_kind so a foreign scale can't be compared", async () => {
+    // The threshold lives on the cosine scale, so only rows that DECLARE that
+    // scale may be compared against it. Without this guard, legacy rows whose
+    // top_score holds an RRF rank score (ceiling ~0.033) compare below any
+    // cosine threshold and flag 100% of scored traffic — the bug this replaces.
+    mockSummaryQueries();
+    await getAnalyticsSummary({});
+
+    const [sql, params] = mockQuery.mock.calls[1];
+    expect(sql).toMatch(/score_kind = \$\d+/);
+    expect(params).toContain(COSINE_SCORE_KIND);
+  });
+
+  it("averages top_score only over cosine-scaled rows in getTopQueries", async () => {
+    // Same scale guard on the dashboard's Avg Cosine column: a legacy RRF row
+    // must not be folded into a mean presented as a cosine similarity.
+    mockQuery.mockResolvedValueOnce({ rows: [] });
+    await getTopQueries(7, 50);
+
+    const [sql, params] = mockQuery.mock.calls[0];
+    expect(sql).toMatch(/avg\(top_score\) FILTER/);
+    expect(sql).toMatch(/score_kind = \$\d+/);
+    expect(params).toContain(COSINE_SCORE_KIND);
+  });
+
   it("threshold constant is 0.5 (matches the brief)", () => {
     expect(LOW_CONFIDENCE_SCORE_THRESHOLD).toBe(0.5);
   });