[pull] main from firecrawl:main

apps/api/package.json

@@ -111,7 +111,7 @@
     "culori": "^4.0.2",
     "dotenv": "^16.3.1",
     "drizzle-orm": "1.0.0-rc.3",
-    "esbuild": "^0.27.2",
+    "esbuild": "^0.28.1",
     "escape-html": "^1.0.3",
     "express": "4.22.0",
     "express-ws": "^5.0.2",
@@ -208,7 +208,8 @@
       "follow-redirects@<1.16.0": ">=1.16.0 <2.0.0",
       "uuid@<11.1.1": "11.1.1",
       "js-cookie@<3.0.7": "3.0.7",
-      "shell-quote@<1.8.4": "1.8.4"
+      "shell-quote@<1.8.4": "1.8.4",
+      "esbuild@>=0.17.0 <0.28.1": "0.28.1"
     }
   },
   "lint-staged": {

apps/api/src/__tests__/snips/index-cache.test.ts

@@ -4,6 +4,9 @@ import {
   filterIndexEntries,
   getCachedIndexEntries,
   getCachedMaxAge,
+  getCachedNegative,
+  isNegativeStillValid,
+  setCachedNegative,
   upsertCachedIndexEntries,
   type IndexCacheEntry,
 } from "../../services/index-cache";
@@ -187,6 +190,36 @@ describe("filterIndexEntries", () => {
   });
 });
 
+describe("isNegativeStillValid", () => {
+  // A marker written at Tq with maxAge Mq stores emptyFrom = Tq - Mq, the left
+  // edge of the confirmed-empty window. A later request is still-empty iff its
+  // own left edge (now - maxAge) is no earlier than emptyFrom.
+  const Tq = 1_000_000_000_000;
+  const Mq = 2 * 24 * 60 * 60 * 1000; // 2 days
+  const emptyFrom = Tq - Mq;
+
+  it("stays valid as time passes with the same maxAge window", () => {
+    // 1 hour later, same 2-day window: left edge moved forward, still >= emptyFrom.
+    expect(isNegativeStillValid(emptyFrom, Mq, Tq + 60 * 60 * 1000)).toBe(true);
+  });
+
+  it("stays valid for a smaller (more recent) window", () => {
+    expect(isNegativeStillValid(emptyFrom, 60 * 60 * 1000, Tq)).toBe(true);
+  });
+
+  it("is invalid for a larger window that looks further back than confirmed", () => {
+    // 14-day window now reaches earlier than the 2-day confirmed-empty left edge.
+    expect(
+      isNegativeStillValid(emptyFrom, 14 * 24 * 60 * 60 * 1000, Tq),
+    ).toBe(false);
+  });
+
+  it("is exactly valid at the boundary", () => {
+    // now - maxAge == emptyFrom
+    expect(isNegativeStillValid(emptyFrom, Mq, Tq)).toBe(true);
+  });
+});
+
 describe("index cache fail-open", () => {
   // Points at a port nothing listens on: every operation must resolve (not
   // throw, not hang) so the read path can fall back to Postgres.
@@ -227,4 +260,16 @@ describe("index cache fail-open", () => {
       getCachedMaxAge(urlHash, undefined, deadClient),
     ).resolves.toBeNull();
   }, 5000);
+
+  it("getCachedNegative resolves to null (disabled or dead) and never throws", async () => {
+    await expect(
+      getCachedNegative("idxc:test", undefined, deadClient),
+    ).resolves.toBeNull();
+  }, 5000);
+
+  it("setCachedNegative resolves without throwing", async () => {
+    await expect(
+      setCachedNegative("idxc:test", Date.now(), undefined, deadClient),
+    ).resolves.toBeUndefined();
+  }, 5000);
 });

apps/api/src/config.ts

@@ -85,6 +85,11 @@ const configSchema = z.object({
   DATABASE_REPLICA_URL: z.string().optional(),
   INDEX_DATABASE_URL: z.string().optional(),
   INDEX_CACHE_REDIS_URL: z.string().optional(),
+  // Negative (miss) caching TTL for index URL->id lookups, in ms. 0 disables
+  // it; the cache then only shields lookups that find data. A positive value
+  // (e.g. 600000 = 10min) also short-circuits repeat lookups for URLs with no
+  // index entry. Kept short so any missed cache-clear self-heals quickly.
+  INDEX_CACHE_NEGATIVE_TTL_MS: z.coerce.number().default(0),
   REDIS_URL: z.string().optional(),
   REDIS_EVICT_URL: z.string().optional(),
   REDIS_RATE_LIMIT_URL: z.string().optional(),
@@ -103,6 +108,9 @@ const configSchema = z.object({
   CLICKHOUSE_ANALYTICS_URL: z.string().optional(),
   CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),
 
+  // Search highlights (beta): semantic highlight model endpoint
+  HIGHLIGHT_MODEL_URL: z.string().optional(),
+
   // Fire Engine
   FIRE_ENGINE_BETA_URL: z.string().optional(),
   FIRE_ENGINE_STAGING_URL: z.string().optional(),

apps/api/src/controllers/v1/types.ts

@@ -1310,6 +1310,7 @@ export type TeamFlags = {
   // POST /v2/search/:jobId/feedback returns 403 TEAM_OPTED_OUT when true.
   searchFeedbackOptOut?: boolean;
   researchBeta?: boolean;
+  highlightsBeta?: boolean;
 } | null;
 
 export type AuthCreditUsageChunkFromTeam = Omit<

apps/api/src/controllers/v2/search.ts

@@ -135,6 +135,7 @@ export async function searchController(
         excludeDomains: req.body.excludeDomains,
         enterprise: req.body.enterprise,
         scrapeOptions: req.body.scrapeOptions,
+        highlights: req.body.highlights,
         timeout: req.body.timeout,
       },
       {

apps/api/src/controllers/v2/types.ts

@@ -1525,6 +1525,7 @@ export type TeamFlags = {
   debugBranding?: boolean;
   maxBrowserSessions?: number;
   researchBeta?: boolean;
+  highlightsBeta?: boolean;
 } | null;
 
 interface RequestWithMaybeACUC<
@@ -1905,6 +1906,10 @@ export const searchRequestSchema = z
     timeout: z.int().positive().finite().prefault(60000),
     ignoreInvalidURLs: z.boolean().optional().prefault(false),
     asyncScraping: z.boolean().optional().prefault(false),
+    // Experimental: replace each result's snippet with query-relevant
+    // highlights pulled from our index (last 30 days), out-of-line from
+    // scrapeURL. Falls back to the provider snippet when the URL isn't indexed.
+    highlights: z.boolean().optional().prefault(false),
     __searchPreviewToken: z.string().optional(),
     scrapeOptions: baseScrapeOptions
       .extend({

apps/api/src/lib/scrape-billing.test.ts

@@ -24,4 +24,66 @@ describe("calculateCreditsToBeBilled", () => {
 
     expect(credits).toBe(30);
   });
+
+  it("bills deterministic JSON at 10 credits when the script was generated", async () => {
+    const credits = await calculateCreditsToBeBilled(
+      {
+        formats: [{ type: "deterministicJson", schema: {} }],
+      } as any,
+      {
+        teamId: "team-id",
+      },
+      {
+        metadata: {
+          statusCode: 200,
+          proxyUsed: "basic",
+        },
+      } as any,
+      {
+        totalCost: 0.01,
+        calls: [
+          {
+            type: "other",
+            model: "vertex/gemini",
+            cost: 0.01,
+            metadata: { module: "deterministic-json", role: "codegen" },
+          },
+        ],
+      } as any,
+      {} as any,
+    );
+
+    expect(credits).toBe(10);
+  });
+
+  it("bills deterministic JSON at 3 credits when a cached script was reused", async () => {
+    const credits = await calculateCreditsToBeBilled(
+      {
+        formats: [{ type: "deterministicJson", schema: {} }],
+      } as any,
+      {
+        teamId: "team-id",
+      },
+      {
+        metadata: {
+          statusCode: 200,
+          proxyUsed: "basic",
+        },
+      } as any,
+      {
+        totalCost: 0.001,
+        calls: [
+          {
+            type: "other",
+            model: "groq/llama",
+            cost: 0.001,
+            metadata: { module: "deterministic-json", role: "askLlm" },
+          },
+        ],
+      } as any,
+      {} as any,
+    );
+
+    expect(credits).toBe(3);
+  });
 });

apps/api/src/lib/scrape-billing.ts

@@ -73,7 +73,14 @@ export async function calculateCreditsToBeBilled(
   }
 
   if (hasFormatOfType(options.formats, "deterministicJson")) {
-    creditsToBeBilled = 7;
+    // 10 when this run generated the extractor script, 3 when it reused a
+    // cached one. The codegen call is tagged in deterministicJson/llm/client.ts.
+    const generatedScript = costTrackingJSON.calls?.some(
+      call =>
+        call.metadata?.module === "deterministic-json" &&
+        call.metadata?.role === "codegen",
+    );
+    creditsToBeBilled = generatedScript ? 10 : 3;
   }
 
   if (

apps/api/src/scraper/scrapeURL/engines/index/index.ts

@@ -20,9 +20,13 @@ import {
   filterIndexEntries,
   getCachedIndexEntries,
   getCachedMaxAge,
+  getCachedNegative,
+  isNegativeStillValid,
   setCachedMaxAge,
+  setCachedNegative,
   upsertCachedIndexEntries,
   useIndexCache,
+  useIndexNegativeCache,
   type IndexCacheEntry,
 } from "../../../../services/index-cache";
 import { indexLookupCounter } from "../../../../lib/index-cache-metrics";
@@ -314,6 +318,7 @@ export async function scrapeURLWithIndex(
   let cacheStatus: "hit" | "filtered" | "miss" | "error" | "disabled" =
     "disabled";
   let servedFromCache = false;
+  let negativeHit = false;
   let timingsCache = 0;
   let timingsDb = 0;
 
@@ -325,8 +330,9 @@ export async function scrapeURLWithIndex(
     dbResult: "hit" | "miss" | "error",
     extra: Record<string, unknown> = {},
   ) => {
-    const outcome =
-      cacheStatus === "hit"
+    const outcome = negativeHit
+      ? "cache_neg_hit"
+      : cacheStatus === "hit"
         ? "cache_hit"
         : cacheStatus === "disabled"
           ? `db_only_${dbResult}`
@@ -384,7 +390,26 @@ export async function scrapeURLWithIndex(
     }
   }
 
-  if (!servedFromCache) {
+  // Negative cache: on a clean positive miss (key absent), a still-valid
+  // negative marker proves there's no index entry for this window, so we can
+  // skip Postgres. Not consulted on "filtered"/"error" (entries exist, or the
+  // cache is unhealthy and we must fall back to the DB), nor for minAge
+  // requests (different no-data semantics — NoCachedDataError, no waterfall).
+  if (
+    !servedFromCache &&
+    useIndexNegativeCache &&
+    cacheStatus === "miss" &&
+    meta.options.minAge === undefined
+  ) {
+    const negStart = Date.now();
+    const neg = await getCachedNegative(variantKey, meta.logger);
+    timingsCache += Date.now() - negStart;
+    if (neg !== null && isNegativeStillValid(neg.emptyFrom, maxAge, Date.now())) {
+      negativeHit = true;
+    }
+  }
+
+  if (!servedFromCache && !negativeHit) {
     const dbStart = Date.now();
     try {
       const rows = await indexGetRecent5({
@@ -418,6 +443,15 @@ export async function scrapeURLWithIndex(
         upsertCachedIndexEntries(variantKey, entries, meta.logger).catch(
           () => {},
         );
+      } else if (
+        useIndexNegativeCache &&
+        rows.length === 0 &&
+        meta.options.minAge === undefined
+      ) {
+        // Confirmed empty for [dbStart - maxAge, dbStart]; record the left edge.
+        setCachedNegative(variantKey, dbStart - maxAge, meta.logger).catch(
+          () => {},
+        );
       }
       data = rows;
     } catch (error) {

apps/api/src/search/execute.ts

@@ -13,6 +13,7 @@ import {
   mergeScrapedContent,
   calculateScrapeCredits,
 } from "./scrape";
+import { applySearchHighlights, highlightsEnvReady } from "./highlights";
 import { trackSearchResults, trackSearchRequest } from "../lib/tracking";
 import type { BillingMetadata } from "../services/billing/types";
 
@@ -30,6 +31,7 @@ interface SearchOptions {
   excludeDomains?: string[];
   enterprise?: ("default" | "anon" | "zdr")[];
   scrapeOptions?: ScrapeOptions;
+  highlights?: boolean;
   timeout: number;
 }
 
@@ -185,6 +187,17 @@ export async function executeSearch(
     }
   }
 
+  // Experimental highlights beta: replace provider snippets with index-backed
+  // highlights. Gated on (1) the request opting in, (2) the team's highlightsBeta
+  // flag, and (3) all required envs being present (index DB, GCS, model). Any
+  // gate failing => silently keep the provider snippets.
+  // Runs after scraping (mergeScrapedContent rebuilds the result objects, so
+  // highlight mutations must come last to survive). Uses the user's original
+  // query, not the domain-filtered upstream query.
+  if (options.highlights && flags?.highlightsBeta === true && highlightsEnvReady()) {
+    await applySearchHighlights(searchResponse, query, logger);
+  }
+
   const scrapeFormats = scrapeOptions?.formats
     ? scrapeOptions.formats.map((f: any) =>
         typeof f === "string" ? f : f.type,