Speed up search by ranking from a title map

src/integrations/pagefind-index.ts

@@ -10,6 +10,8 @@
 import type { AstroIntegration } from 'astro';
 import { fileURLToPath } from 'node:url';
 import * as path from 'node:path';
+import { gunzipSync } from 'node:zlib';
+import { readdir, readFile, writeFile } from 'node:fs/promises';
 // Statically, because `astro:build:done` fires after Vite's module runner has
 // closed and a dynamic import from inside the hook cannot be resolved.
 import { createIndex, close } from 'pagefind';
@@ -81,10 +83,105 @@ export default function pagefindIndex(): AstroIntegration {
           // Files scanned, not pages indexed: the redirect stubs are counted
           // here and then dropped for having no article.
           logger.info(`scanned ${added.page_count} pages into docs/pagefind`);
+
+          const titles = await writeTitleMap(
+            path.join(distDir, 'docs', 'pagefind')
+          );
+          logger.info(`wrote ${titles.pages} titles to ${titles.file}`);
         } finally {
           await close();
         }
       },
     },
   };
 }
+
+/**
+ * The map's name, carrying the index's own hash. Pagefind hashes its chunks so a
+ * cache cannot serve one build's index against another's, and a map read against
+ * the wrong index joins against nothing. The client reads the hash out of
+ * `pagefind-entry.json` to build the same name.
+ *
+ * A module, so that Front Door's static-content rule caches it for a week as
+ * immutable and compressed. That rule lists extensions, and a `.json` name falls
+ * through to `no-cache`, where an ETag-less revalidation costs a full
+ * re-download — paid on every page that loads the overlay.
+ */
+const titleMapName = (hash: string) => `docs-titles.${hash}.js`;
+
+// Pagefind prefixes every decompressed chunk with this before the JSON.
+const FRAGMENT_MAGIC = 'pagefind_dcd';
+
+/**
+ * Writes what the overlay needs to rank a result without fetching it.
+ *
+ * Ranking needs a URL and a title, and Pagefind keeps both in the per-page
+ * fragment — so ranking thirty results meant fetching thirty files, and a
+ * landing page ranked past that could not be reached at all. A search result
+ * stub carries the id of its own fragment, so one map from id to url and title
+ * lets the whole result set be ranked from a single file.
+ *
+ * Read back out of the fragments rather than collected during indexing, because
+ * the ids are assigned by Pagefind as it writes them.
+ */
+async function writeTitleMap(
+  pagefindDir: string
+): Promise<{ pages: number; file: string }> {
+  const entry = JSON.parse(
+    await readFile(path.join(pagefindDir, 'pagefind-entry.json'), 'utf8')
+  ) as { languages: Record<string, { hash: string }> };
+
+  const languages = Object.keys(entry.languages ?? {});
+  // One map covers every fragment, so it can only carry one language's hash. A
+  // second language would need one map each, keyed the way Pagefind keys its own
+  // chunks — worth failing loudly over rather than shipping a map the client
+  // looks for under the wrong name.
+  if (languages.length !== 1) {
+    throw new Error(
+      `expected one indexed language, found ${languages.length || 'none'}: the title map is named after the index hash and cannot cover several`
+    );
+  }
+
+  const file = titleMapName(entry.languages[languages[0]].hash);
+  const dir = path.join(pagefindDir, 'fragment');
+  const names = (await readdir(dir)).filter((name) =>
+    name.endsWith('.pf_fragment')
+  );
+
+  const map: Record<string, [url: string, title: string]> = {};
+
+  for (const name of names) {
+    const raw = gunzipSync(await readFile(path.join(dir, name))).toString(
+      'utf8'
+    );
+
+    // Checked before parsing: a Pagefind release that changes the chunk format
+    // has to fail the build here, rather than write a map the overlay silently
+    // cannot join against.
+    if (!raw.startsWith(FRAGMENT_MAGIC)) {
+      throw new Error(
+        `unexpected fragment format in ${name}: Pagefind's own prefix is missing, so the title map cannot be trusted`
+      );
+    }
+
+    const fragment = JSON.parse(raw.slice(raw.indexOf('{'))) as {
+      url: string;
+      meta?: Record<string, string>;
+    };
+
+    // The stub's `id` is the filename without its extension, which is the join.
+    map[path.basename(name, '.pf_fragment')] = [
+      fragment.url,
+      fragment.meta?.title ?? '',
+    ];
+  }
+
+  await writeFile(
+    path.join(pagefindDir, file),
+    `export default ${JSON.stringify(map)};
+`,
+    'utf8'
+  );
+
+  return { pages: names.length, file };
+}

src/layouts/Api.astro

@@ -99,7 +99,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter);
               /* Copy as markdown temporarily disabled until we can get it working with the API docs. Deliberately no "Edit on GitHub" because these are generated and should not be hand edited */
             }
           </div>
-          <div class="page-content anim-show-parent" {...searchIndex.content}>
+          <div class="page-content anim-show-parent">
             <slot />
             <Authors frontmatter={frontmatter} lang={lang} />
             <Taxonomy frontmatter={frontmatter} lang={lang} />

src/layouts/Default.astro

@@ -100,7 +100,7 @@ const searchIndex = searchIndexAttributes(Astro.url.pathname, frontmatter);
               lang={lang}
             />
           </div>
-          <div class="page-content anim-show-parent" {...searchIndex.content}>
+          <div class="page-content anim-show-parent">
             <slot />
             <Authors frontmatter={frontmatter} lang={lang} />
             <Taxonomy frontmatter={frontmatter} lang={lang} />

src/lib/searchIndexing.ts

@@ -13,30 +13,12 @@ type ArticleAttributes = {
   'data-pagefind-default-meta'?: string;
 };
 
-/**
- * A second filter, on an element inside the article rather than on the article
- * itself: Pagefind reads one `key:value` per `data-pagefind-filter`, and a
- * comma-separated pair is taken as a single value.
- */
-type ContentAttributes = {
-  'data-pagefind-filter'?: string;
-};
-
 type IndexAttributes = {
   article: ArticleAttributes;
-  content: ContentAttributes;
 };
 
 /**
- * How shallow a page has to be to count as one a reader might name. Two segments
- * past `/docs/`, which covers `/docs/deployments/` and
- * `/docs/infrastructure/deployment-targets/` but not the pages inside them.
- */
-const LANDING_DEPTH = 3;
-
-/**
- * The `data-pagefind-*` attributes for a page: `article` spreads onto the
- * `<article>`, `content` onto the page content inside it.
+ * The `data-pagefind-*` attributes for a page, to spread onto the `<article>`.
  *
  * `navSearch` rather than `PostFiltering.showInSearch`, which also hides a page
  * with a future `pubDate`, a `draft: true` and a `listable: false`: a page that
@@ -53,13 +35,7 @@ export function searchIndexAttributes(
 
   // `all` rather than the default `index`: a bare ignore still lets Pagefind
   // read a title or metadata out of the block.
-  if (!indexable)
-    return { article: { 'data-pagefind-ignore': 'all' }, content: {} };
-
-  // Marks the pages the overlay's second, narrowed search looks through. Only
-  // the shallow pages carry it, so the filter chunk stays small and that search
-  // has a few hundred candidates rather than the whole site.
-  const isLanding = pathname.split('/').filter(Boolean).length <= LANDING_DEPTH;
+  if (!indexable) return { article: { 'data-pagefind-ignore': 'all' } };
 
   return {
     article: {
@@ -75,6 +51,5 @@ export function searchIndexAttributes(
         ? { 'data-pagefind-default-meta': `title:${frontmatter.title}` }
         : {}),
     },
-    content: isLanding ? { 'data-pagefind-filter': 'landing:true' } : {},
   };
 }