From 8d1cfedaaddb40943b4f241dc4ef83b6ea2835e3 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 01:59:55 +0000 Subject: [PATCH] fix(docs): order llms-full.txt by the navigation tree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `llms-full.txt` enumerated `source.getPages(LANG)` and joined the result unsorted, so page order was whatever order the loader finished reading files in. That is stable inside one build and not across builds: two consecutive builds of an untouched tree emitted the same 79 pages with the whole `configure/` group and the whole `deploy/` group swapped. Order by `source.getPageTree(LANG)` — `meta.json` order, the site navigation — rather than by a sort. Sorting would be deterministic and worse: the order pages appear in this file is read, and alphabetical would scatter `build/`, `deploy/` and `operate/` through a flat A-to-Z list. `llms.txt` already walks the tree, so the index and the full text now agree on sequence. Reproducibility falls out of that. Membership is unchanged: same 79 page blocks, same 93 `#`-prefixed lines, same 798 heading lines, same 646 link targets, and the sorted line multiset of the served body is byte-identical to the old one. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01G5zjYc2BoFV2NjKBBapC7C --- apps/docs/app/llms-full.txt/route.ts | 71 +++++++++++++++++++++++----- 1 file changed, 60 insertions(+), 11 deletions(-) diff --git a/apps/docs/app/llms-full.txt/route.ts b/apps/docs/app/llms-full.txt/route.ts index 722f147..7657a86 100644 --- a/apps/docs/app/llms-full.txt/route.ts +++ b/apps/docs/app/llms-full.txt/route.ts @@ -1,3 +1,4 @@ +import { flattenTree } from 'fumadocs-core/page-tree'; import { i18n } from '@/lib/i18n'; import { SITE_URL } from '@/lib/seo'; import { getLLMText, source } from '@/lib/source'; @@ -7,12 +8,14 @@ export const revalidate = false; /** * English only, by `AGENTS.md` rule 1 — English is the single source of truth * and every `*..mdx` is a derived artifact. This is the same pin - * `llms.txt` carries, for the same reason: `source.getPages()` filters by - * locale **only when a language is passed** — called bare it returns every - * language's page set concatenated, so this route used to emit all 335 pages - * across 7 locales (4.6 MB, the same page appearing up to seven times) instead - * of the 79 English ones. The locale text stays reachable per page through the - * `.mdx` rewrite and the locale URLs announced at the end of `llms.txt`. + * `llms.txt` carries, for the same reason: `source.getPages()` and + * `source.getPageTree()` alike filter by locale **only when a language is + * passed** — called bare they list every language, which is how this route + * once emitted all 335 pages across 7 locales (4.6 MB, the same page appearing + * up to seven times) instead of the 79 English ones. Which of the two this + * route enumerates has changed; that the argument is mandatory has not. The + * locale text stays reachable per page through the `.mdx` rewrite and the + * locale URLs announced at the end of `llms.txt`. */ const LANG = i18n.defaultLanguage; @@ -71,17 +74,63 @@ function absoluteLinks(text: string, pageUrl: string): string { return lines.join('\n'); } +/** + * Every English page, in the order the site navigation puts them. + * + * `source.getPages(LANG)` — what this route used to enumerate — hands back + * pages in whatever order the loader finished reading them. That is stable + * within one build and not across builds: two consecutive builds of an + * untouched tree emitted this file with the whole `configure/` group and the + * whole `deploy/` group swapped, so the bytes changed for the same 79 pages + * and nothing in the repository had moved. + * + * The repair is not "sort it". Unlike `sitemap.xml`, where entry order carries + * nothing to a crawler, **the order pages appear in here is read** — this file + * is one long document. So a deterministic order is necessary and not + * sufficient: alphabetical would be reproducible and *worse* than the accident + * it replaced, scattering `build/`, `deploy/` and `operate/` through a flat + * A-to-Z list. + * + * `source.getPageTree(LANG)` is `meta.json` order, which is the site + * navigation, which is the order a reader of these pages already knows. It is + * also what `llms.txt` already walks, so the index and the full text now agree + * on sequence instead of contradicting each other. Reproducibility follows + * from fixing the meaning rather than the other way round. + * + * Membership is unchanged. `flattenTree` yields a folder's index page followed + * by its children, recursively, and every one of the 79 English pages is + * reachable from a `meta.json`. What decides membership is now the tree rather + * than the file system — the same rule the sidebar and `llms.txt` already + * follow, so a page that no `meta.json` lists is absent from all three + * together rather than from two of them quietly. + */ +function navigationPages() { + return flattenTree(source.getPageTree(LANG).children).map((node) => { + const page = source.getNodePage(node, LANG); + + // Unreachable unless fumadocs hands back a page node whose `$ref` resolves + // to nothing. Loud on purpose: the alternative is a page dropping out of + // this file without a trace, which is a worse defect than the ordering one + // this function exists to fix. + if (!page) { + throw new Error( + `llms-full.txt: page tree node ${String(node.$id ?? node.url)} resolves to no page`, + ); + } + + return page; + }); +} + export async function GET() { // Each page is rewritten against its own URL rather than the joined body // against the site root: `./objectql` means something different on // `/docs/reference/cel` than it does three sections away, and a page whose // text left a code fence unclosed cannot then leak that state into the next // page's links. - const scan = source - .getPages(LANG) - .map(async (page) => - absoluteLinks(await getLLMText(page), `${SITE_URL}${page.url}`), - ); + const scan = navigationPages().map(async (page) => + absoluteLinks(await getLLMText(page), `${SITE_URL}${page.url}`), + ); const scanned = await Promise.all(scan); return new Response(scanned.join('\n\n'));