From 63aac03d1a1ca25c86f3b95013a80b50b8e7c622 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Fri, 12 Jun 2026 14:06:25 -0700 Subject: [PATCH 1/5] feat(api): negative caching for index URL->id lookups (v2) (#3775) Co-authored-by: Claude Fable 5 --- .../src/__tests__/snips/index-cache.test.ts | 45 +++++++++ apps/api/src/config.ts | 5 + .../scraper/scrapeURL/engines/index/index.ts | 40 +++++++- apps/api/src/services/index-cache.ts | 91 ++++++++++++++++++- 4 files changed, 177 insertions(+), 4 deletions(-) diff --git a/apps/api/src/__tests__/snips/index-cache.test.ts b/apps/api/src/__tests__/snips/index-cache.test.ts index 12e0496e80..e1bb881eb9 100644 --- a/apps/api/src/__tests__/snips/index-cache.test.ts +++ b/apps/api/src/__tests__/snips/index-cache.test.ts @@ -4,6 +4,9 @@ import { filterIndexEntries, getCachedIndexEntries, getCachedMaxAge, + getCachedNegative, + isNegativeStillValid, + setCachedNegative, upsertCachedIndexEntries, type IndexCacheEntry, } from "../../services/index-cache"; @@ -187,6 +190,36 @@ describe("filterIndexEntries", () => { }); }); +describe("isNegativeStillValid", () => { + // A marker written at Tq with maxAge Mq stores emptyFrom = Tq - Mq, the left + // edge of the confirmed-empty window. A later request is still-empty iff its + // own left edge (now - maxAge) is no earlier than emptyFrom. + const Tq = 1_000_000_000_000; + const Mq = 2 * 24 * 60 * 60 * 1000; // 2 days + const emptyFrom = Tq - Mq; + + it("stays valid as time passes with the same maxAge window", () => { + // 1 hour later, same 2-day window: left edge moved forward, still >= emptyFrom. + expect(isNegativeStillValid(emptyFrom, Mq, Tq + 60 * 60 * 1000)).toBe(true); + }); + + it("stays valid for a smaller (more recent) window", () => { + expect(isNegativeStillValid(emptyFrom, 60 * 60 * 1000, Tq)).toBe(true); + }); + + it("is invalid for a larger window that looks further back than confirmed", () => { + // 14-day window now reaches earlier than the 2-day confirmed-empty left edge. + expect( + isNegativeStillValid(emptyFrom, 14 * 24 * 60 * 60 * 1000, Tq), + ).toBe(false); + }); + + it("is exactly valid at the boundary", () => { + // now - maxAge == emptyFrom + expect(isNegativeStillValid(emptyFrom, Mq, Tq)).toBe(true); + }); +}); + describe("index cache fail-open", () => { // Points at a port nothing listens on: every operation must resolve (not // throw, not hang) so the read path can fall back to Postgres. @@ -227,4 +260,16 @@ describe("index cache fail-open", () => { getCachedMaxAge(urlHash, undefined, deadClient), ).resolves.toBeNull(); }, 5000); + + it("getCachedNegative resolves to null (disabled or dead) and never throws", async () => { + await expect( + getCachedNegative("idxc:test", undefined, deadClient), + ).resolves.toBeNull(); + }, 5000); + + it("setCachedNegative resolves without throwing", async () => { + await expect( + setCachedNegative("idxc:test", Date.now(), undefined, deadClient), + ).resolves.toBeUndefined(); + }, 5000); }); diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts index c3e1872c05..fa52802ade 100644 --- a/apps/api/src/config.ts +++ b/apps/api/src/config.ts @@ -85,6 +85,11 @@ const configSchema = z.object({ DATABASE_REPLICA_URL: z.string().optional(), INDEX_DATABASE_URL: z.string().optional(), INDEX_CACHE_REDIS_URL: z.string().optional(), + // Negative (miss) caching TTL for index URL->id lookups, in ms. 0 disables + // it; the cache then only shields lookups that find data. A positive value + // (e.g. 600000 = 10min) also short-circuits repeat lookups for URLs with no + // index entry. Kept short so any missed cache-clear self-heals quickly. + INDEX_CACHE_NEGATIVE_TTL_MS: z.coerce.number().default(0), REDIS_URL: z.string().optional(), REDIS_EVICT_URL: z.string().optional(), REDIS_RATE_LIMIT_URL: z.string().optional(), diff --git a/apps/api/src/scraper/scrapeURL/engines/index/index.ts b/apps/api/src/scraper/scrapeURL/engines/index/index.ts index eb499ed67b..0f607d1f57 100644 --- a/apps/api/src/scraper/scrapeURL/engines/index/index.ts +++ b/apps/api/src/scraper/scrapeURL/engines/index/index.ts @@ -20,9 +20,13 @@ import { filterIndexEntries, getCachedIndexEntries, getCachedMaxAge, + getCachedNegative, + isNegativeStillValid, setCachedMaxAge, + setCachedNegative, upsertCachedIndexEntries, useIndexCache, + useIndexNegativeCache, type IndexCacheEntry, } from "../../../../services/index-cache"; import { indexLookupCounter } from "../../../../lib/index-cache-metrics"; @@ -314,6 +318,7 @@ export async function scrapeURLWithIndex( let cacheStatus: "hit" | "filtered" | "miss" | "error" | "disabled" = "disabled"; let servedFromCache = false; + let negativeHit = false; let timingsCache = 0; let timingsDb = 0; @@ -325,8 +330,9 @@ export async function scrapeURLWithIndex( dbResult: "hit" | "miss" | "error", extra: Record = {}, ) => { - const outcome = - cacheStatus === "hit" + const outcome = negativeHit + ? "cache_neg_hit" + : cacheStatus === "hit" ? "cache_hit" : cacheStatus === "disabled" ? `db_only_${dbResult}` @@ -384,7 +390,26 @@ export async function scrapeURLWithIndex( } } - if (!servedFromCache) { + // Negative cache: on a clean positive miss (key absent), a still-valid + // negative marker proves there's no index entry for this window, so we can + // skip Postgres. Not consulted on "filtered"/"error" (entries exist, or the + // cache is unhealthy and we must fall back to the DB), nor for minAge + // requests (different no-data semantics — NoCachedDataError, no waterfall). + if ( + !servedFromCache && + useIndexNegativeCache && + cacheStatus === "miss" && + meta.options.minAge === undefined + ) { + const negStart = Date.now(); + const neg = await getCachedNegative(variantKey, meta.logger); + timingsCache += Date.now() - negStart; + if (neg !== null && isNegativeStillValid(neg.emptyFrom, maxAge, Date.now())) { + negativeHit = true; + } + } + + if (!servedFromCache && !negativeHit) { const dbStart = Date.now(); try { const rows = await indexGetRecent5({ @@ -418,6 +443,15 @@ export async function scrapeURLWithIndex( upsertCachedIndexEntries(variantKey, entries, meta.logger).catch( () => {}, ); + } else if ( + useIndexNegativeCache && + rows.length === 0 && + meta.options.minAge === undefined + ) { + // Confirmed empty for [dbStart - maxAge, dbStart]; record the left edge. + setCachedNegative(variantKey, dbStart - maxAge, meta.logger).catch( + () => {}, + ); } data = rows; } catch (error) { diff --git a/apps/api/src/services/index-cache.ts b/apps/api/src/services/index-cache.ts index 16a8fa859f..3786d526ee 100644 --- a/apps/api/src/services/index-cache.ts +++ b/apps/api/src/services/index-cache.ts @@ -16,6 +16,7 @@ import type { Logger } from "winston"; // instance. const ENTRY_KEY_PREFIX = "idxc:"; +const NEG_KEY_PREFIX = "idxcneg:"; const MAX_AGE_KEY_PREFIX = "idxma:"; const ENTRY_TTL_SECONDS = 7 * 24 * 60 * 60; const MAX_AGE_TTL_SECONDS = 15 * 60; @@ -43,6 +44,18 @@ indexCacheRedis?.on("error", error => { export const useIndexCache = indexCacheRedis !== null; +// Negative (miss) caching is opt-in via a positive TTL. When enabled, a +// confirmed DB miss is recorded so repeat lookups for URLs with no index +// entry can skip Postgres entirely. +const NEGATIVE_TTL_MS = config.INDEX_CACHE_NEGATIVE_TTL_MS; +export const useIndexNegativeCache = useIndexCache && NEGATIVE_TTL_MS > 0; + +// The negative marker shares the variant's hash but lives under its own +// prefix and TTL (deriveIndexVariantKey returns "idxc:"). +function negKeyFor(variantKey: string): string { + return NEG_KEY_PREFIX + variantKey.slice(ENTRY_KEY_PREFIX.length); +} + export type IndexCacheEntry = { id: string; created_at: string; @@ -203,9 +216,13 @@ export async function upsertCachedIndexEntries( const pipeline = client.pipeline(); pipeline.hset(key, fields); pipeline.expire(key, ENTRY_TTL_SECONDS); + // Writing positive entries invalidates any negative marker for this + // variant — this is what keeps a surviving negative marker a proof that + // nothing was inserted since it was set. + pipeline.del(negKeyFor(key)); pipeline.hlen(key); const results = await pipeline.exec(); - const hlen = results?.[2]?.[1]; + const hlen = results?.[3]?.[1]; if (typeof hlen === "number" && hlen > ENTRY_CAP) { const raw = await client.hgetall(key); const parsed = Object.entries(raw) @@ -314,3 +331,75 @@ export async function setCachedMaxAge( }); } } + +// A negative marker records that, as of when it was written, there was no +// index entry in [emptyFrom, writeTime] for the variant. A later lookup for +// window [now - maxAgeMs, now] is still guaranteed empty iff its left edge is +// no earlier than emptyFrom AND the marker still exists (a positive write +// would have deleted it, covering the (writeTime, now] tail). So the only +// check needed at read time is the left-edge comparison. +export function isNegativeStillValid( + emptyFrom: number, + maxAgeMs: number, + now: number, +): boolean { + return now - maxAgeMs >= emptyFrom; +} + +export async function getCachedNegative( + variantKey: string, + logger: Logger = _logger, + client: IORedis | null = indexCacheRedis, +): Promise<{ emptyFrom: number } | null> { + if (client === null || !useIndexNegativeCache) { + return null; + } + try { + const raw = await withTimeout( + client.get(negKeyFor(variantKey)), + READ_TIMEOUT_MS, + ); + if (raw === TIMED_OUT) { + indexCacheErrorCounter.inc({ op: "negative_read_timeout" }); + return null; + } + if (raw === null || raw === undefined) { + return null; + } + const emptyFrom = JSON.parse(raw).emptyFrom; + return typeof emptyFrom === "number" ? { emptyFrom } : null; + } catch (error) { + indexCacheErrorCounter.inc({ op: "negative_read" }); + logger.warn("Index cache negative read failed", { + module: "index-cache", + error, + }); + return null; + } +} + +// emptyFrom is the left edge of the confirmed-empty window: queryTime - maxAge. +export async function setCachedNegative( + variantKey: string, + emptyFrom: number, + logger: Logger = _logger, + client: IORedis | null = indexCacheRedis, +): Promise { + if (client === null || !useIndexNegativeCache) { + return; + } + try { + await client.set( + negKeyFor(variantKey), + JSON.stringify({ emptyFrom }), + "PX", + NEGATIVE_TTL_MS, + ); + } catch (error) { + indexCacheErrorCounter.inc({ op: "negative_write" }); + logger.warn("Index cache negative write failed", { + module: "index-cache", + error, + }); + } +} From 6b7c5e9cf27ce2ccb03107be338600fad4df5448 Mon Sep 17 00:00:00 2001 From: tomsideguide Date: Fri, 12 Jun 2026 17:49:26 -0400 Subject: [PATCH 2/5] feat(api/billing): bill deterministic json at 10 credits on codegen, 3 on cached script (#3750) --- apps/api/src/lib/scrape-billing.test.ts | 62 +++++++++++++++++++++++++ apps/api/src/lib/scrape-billing.ts | 9 +++- 2 files changed, 70 insertions(+), 1 deletion(-) diff --git a/apps/api/src/lib/scrape-billing.test.ts b/apps/api/src/lib/scrape-billing.test.ts index 9f4c72cb24..70361b069f 100644 --- a/apps/api/src/lib/scrape-billing.test.ts +++ b/apps/api/src/lib/scrape-billing.test.ts @@ -24,4 +24,66 @@ describe("calculateCreditsToBeBilled", () => { expect(credits).toBe(30); }); + + it("bills deterministic JSON at 10 credits when the script was generated", async () => { + const credits = await calculateCreditsToBeBilled( + { + formats: [{ type: "deterministicJson", schema: {} }], + } as any, + { + teamId: "team-id", + }, + { + metadata: { + statusCode: 200, + proxyUsed: "basic", + }, + } as any, + { + totalCost: 0.01, + calls: [ + { + type: "other", + model: "vertex/gemini", + cost: 0.01, + metadata: { module: "deterministic-json", role: "codegen" }, + }, + ], + } as any, + {} as any, + ); + + expect(credits).toBe(10); + }); + + it("bills deterministic JSON at 3 credits when a cached script was reused", async () => { + const credits = await calculateCreditsToBeBilled( + { + formats: [{ type: "deterministicJson", schema: {} }], + } as any, + { + teamId: "team-id", + }, + { + metadata: { + statusCode: 200, + proxyUsed: "basic", + }, + } as any, + { + totalCost: 0.001, + calls: [ + { + type: "other", + model: "groq/llama", + cost: 0.001, + metadata: { module: "deterministic-json", role: "askLlm" }, + }, + ], + } as any, + {} as any, + ); + + expect(credits).toBe(3); + }); }); diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts index 3efa050458..2de00540a2 100644 --- a/apps/api/src/lib/scrape-billing.ts +++ b/apps/api/src/lib/scrape-billing.ts @@ -73,7 +73,14 @@ export async function calculateCreditsToBeBilled( } if (hasFormatOfType(options.formats, "deterministicJson")) { - creditsToBeBilled = 7; + // 10 when this run generated the extractor script, 3 when it reused a + // cached one. The codegen call is tagged in deterministicJson/llm/client.ts. + const generatedScript = costTrackingJSON.calls?.some( + call => + call.metadata?.module === "deterministic-json" && + call.metadata?.role === "codegen", + ); + creditsToBeBilled = generatedScript ? 10 : 3; } if ( From 8c6de40e336d949d082360d4fab05d71cfd86430 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gerg=C5=91=20M=C3=B3ricz?= Date: Fri, 12 Jun 2026 14:59:00 -0700 Subject: [PATCH 3/5] chore(api): experimental (WIP) (#3776) Co-authored-by: Claude Opus 4.8 (1M context) --- apps/api/src/search/highlight-model.ts | 203 +++++++++++++++++++++++++ apps/api/src/search/highlights.ts | 187 +++++++++++++++++++++++ 2 files changed, 390 insertions(+) create mode 100644 apps/api/src/search/highlight-model.ts create mode 100644 apps/api/src/search/highlights.ts diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts new file mode 100644 index 0000000000..817c37f038 --- /dev/null +++ b/apps/api/src/search/highlight-model.ts @@ -0,0 +1,203 @@ +import type { Logger } from "winston"; +import { config } from "../config"; + +// Semantic highlight model: it scores each sentence of a context against a +// question (semantic similarity), no LLM. Inference is fast (~10ms), but the +// model caps context at ~4k tokens, so we chunk the page, score every chunk, +// pool the per-sentence scores globally, and keep the best sentences. Endpoint +// is config-gated (HIGHLIGHT_MODEL_URL); callers must confirm it's set via +// highlightsEnvReady() before invoking. + +// ~4 chars/token; 10k chars (~2.5k tokens) leaves headroom under the 4k cap for +// the question + tokenizer variance (markdown/code tokenizes denser than prose). +const CHUNK_CHARS = 10000; +// Bound load for very long pages (chunks run concurrently per result). +const MAX_CHUNKS = 10; +// Keep sentences scoring at/above this (matches the model's default threshold). +const SELECT_THRESHOLD = 0.5; +// Cap the assembled snippet length. +const MAX_SELECTED = 12; +const REQUEST_TIMEOUT_MS = 30000; + +interface ScoredSentence { + text: string; + score: number; + order: number; // global document order +} + +// Strip markdown syntax so a highlight reads as plain prose: links/images keep +// their text, bare URLs and emphasis/heading/code marks go, line-number gutters +// from rendered code blocks are dropped, whitespace collapses. +function cleanSentence(text: string): string { + return text + .replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // links -> link text + .replace(/https?:\/\/\S+/g, " ") // bare URLs + .replace(/^\s*\d+\s*/, "") // leading code-block line number + .replace(/[#*`>_~|\\]+/g, " ") // markdown marks + .replace(/\s+/g, " ") + .trim(); +} + +// Drop markdown/code artifacts (line numbers, "=====", "//", "[ 01 / 06 ]") +// that the semantic model sometimes scores highly. Real sentences have prose. +function isContentful(text: string): boolean { + const letters = (text.match(/[a-zA-Z]/g) ?? []).length; + return letters >= 15; +} + +interface HighlightModelResponse { + kept_sentences?: string[]; + sentence_probabilities?: number[]; +} + +/** + * Split markdown into chunks under the model's token budget. Splits on blank + * lines (paragraph boundaries) to avoid cutting mid-sentence; oversized + * paragraphs are hard-split as a last resort. Capped at MAX_CHUNKS. + */ +function chunkMarkdown(markdown: string): string[] { + const chunks: string[] = []; + let current = ""; + + const flush = () => { + if (current.trim() !== "") chunks.push(current); + current = ""; + }; + + for (const para of markdown.split(/\n{2,}/)) { + if (chunks.length >= MAX_CHUNKS) break; + + if (para.length > CHUNK_CHARS) { + flush(); + for ( + let i = 0; + i < para.length && chunks.length < MAX_CHUNKS; + i += CHUNK_CHARS + ) { + chunks.push(para.slice(i, i + CHUNK_CHARS)); + } + continue; + } + + if (current.length + para.length + 2 > CHUNK_CHARS) { + flush(); + } + current += (current === "" ? "" : "\n\n") + para; + } + + if (chunks.length < MAX_CHUNKS) flush(); + return chunks.slice(0, MAX_CHUNKS); +} + +async function scoreChunk( + question: string, + context: string, + logger: Logger, +): Promise<{ text: string; score: number }[]> { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); + try { + const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/v1/highlight`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + question, + context, + // threshold 0 => every sentence is returned with its score, so we can + // pool scores across chunks and select globally. + threshold: 0.0, + language: "auto", + return_sentence_metrics: true, + }), + signal: controller.signal, + }); + + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error( + `highlight model HTTP ${res.status}: ${body.slice(0, 200)}`, + ); + } + + const data = (await res.json()) as HighlightModelResponse; + const sentences = data.kept_sentences ?? []; + const probs = data.sentence_probabilities ?? []; + const n = Math.min(sentences.length, probs.length); + const out: { text: string; score: number }[] = []; + for (let i = 0; i < n; i++) { + out.push({ text: sentences[i], score: probs[i] }); + } + return out; + } finally { + clearTimeout(timer); + } +} + +/** + * Generate query-relevant highlights from page markdown using the semantic + * highlight model. Returns the assembled highlight string (best sentences in + * document order), or null if nothing clears the threshold / all chunks fail. + */ +export async function generateSemanticHighlights( + markdown: string, + query: string, + opts: { logger: Logger }, +): Promise { + const chunks = chunkMarkdown(markdown); + if (chunks.length === 0) return null; + + const start = Date.now(); + const perChunk = await Promise.all( + chunks.map(async (chunk, idx) => { + try { + return await scoreChunk(query, chunk, opts.logger); + } catch (error) { + opts.logger.warn("highlight model chunk failed", { + error: error instanceof Error ? error.message : String(error), + chunkIdx: idx, + }); + return []; + } + }), + ); + + // Pool every scored sentence, preserving document order (Promise.all keeps + // chunk order, and sentences within a chunk are already in order). + const all: ScoredSentence[] = []; + let order = 0; + for (const chunkSentences of perChunk) { + for (const s of chunkSentences) { + const idx = order++; + const cleaned = cleanSentence(s.text); + if (isContentful(cleaned)) { + all.push({ text: cleaned, score: s.score, order: idx }); + } + } + } + if (all.length === 0) return null; + + // Pick the best: keep sentences at/above the threshold, cap to the top + // MAX_SELECTED by score, then restore document order for readability. + let selected = all.filter(s => s.score >= SELECT_THRESHOLD); + if (selected.length === 0) return null; + selected.sort((a, b) => b.score - a.score); + selected = selected.slice(0, MAX_SELECTED); + selected.sort((a, b) => a.order - b.order); + + const text = selected + .map(s => s.text.trim()) + .join(" ") + .replace(/\s+/g, " ") + .trim(); + + opts.logger.info("semantic highlights generated", { + chunks: chunks.length, + scoredSentences: all.length, + selected: selected.length, + topScore: all.reduce((m, s) => Math.max(m, s.score), 0), + elapsedMs: Date.now() - start, + }); + + return text === "" ? null : text; +} diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts new file mode 100644 index 0000000000..116f5ff0d0 --- /dev/null +++ b/apps/api/src/search/highlights.ts @@ -0,0 +1,187 @@ +import type { Logger } from "winston"; +import { SearchV2Response } from "../lib/entities"; +import { + normalizeURLForIndex, + hashURL, + getIndexFromGCS, + useIndex, +} from "../services"; +import { indexGetRecent5 } from "../db/rpc"; +import { parseMarkdown } from "../lib/html-to-markdown"; +import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements"; +import type { ScrapeOptions } from "../controllers/v2/types"; +import { generateSemanticHighlights } from "./highlight-model"; +import { config } from "../config"; + +// How far back into the index we're willing to reach for highlight source text. +const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days + +/** + * Whether the deployment has every dependency the highlights beta needs: the + * index DB (to find cached content), the GCS index bucket (to fetch it), and the + * highlight model endpoint (to score it). Missing any => silently skip. + */ +export function highlightsEnvReady(): boolean { + return ( + useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL + ); +} + +// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind +// this many more-recent error entries, in which case we surface the newest one. +const ERROR_COUNT_TO_REGISTER = 3; + +// This whole module runs out-of-line from scrapeURL on purpose: it reads +// already-indexed content directly from the index DB + GCS instead of routing +// through the scrape engine. That keeps highlight generation off the critical +// scrape path and lets us experiment with latency freely. + +/** + * Fetch the most recent indexed markdown for a URL within the last 30 days. + * Returns null when the URL isn't in the index (or the lookup fails) so callers + * can fall back to the provider snippet. + */ +async function getIndexedMarkdownForURL( + url: string, + logger: Logger, +): Promise { + if (!useIndex) { + return null; + } + + try { + const normalizedURL = normalizeURLForIndex(url); + const urlHash = hashURL(normalizedURL); + + // Match the most common index variant (default scrape options) to maximize + // hit rate: desktop, ads blocked, no screenshot, no location, no stealth. + const rows = await indexGetRecent5({ + url_hash: urlHash, + max_age_ms: HIGHLIGHTS_INDEX_MAX_AGE_MS, + is_mobile: false, + block_ads: true, + feature_screenshot: false, + feature_screenshot_fullscreen: false, + location_country: null, + location_languages: null, + wait_time_ms: 0, + is_stealth: false, + min_age_ms: null, + }); + + if (!rows || rows.length === 0) { + return null; + } + + const newest200Index = rows.findIndex( + x => x.status >= 200 && x.status < 300, + ); + const selected = + newest200Index >= ERROR_COUNT_TO_REGISTER || newest200Index === -1 + ? rows[0] + : rows[newest200Index]; + + const doc = await getIndexFromGCS( + selected.id + ".json", + logger.child({ module: "search/highlights", method: "getIndexFromGCS" }), + ); + if (!doc || !doc.html) { + return null; + } + + // Skip raw base64 PDFs — they aren't useful as highlight source text. + if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) { + return null; + } + + // The index stores rawHtml, so we must run the same cleaning the scrape + // pipeline does (strip