From 9073d14876a616c669a317e8ce1169f5ef850121 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 01:23:19 +0000 Subject: [PATCH] fix(docs): emit absolute URLs from llms.txt and llms-full.txt MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both llms endpoints emitted site-relative links. That file is fetched and its text lifted into a context window where the origin is no longer attached to it, so a relative path resolves only if whatever moved it there also carried the base URL — which is exactly what a consumer of an llms.txt will not do. The marketing site's own /llms.txt has always emitted absolute URLs; this is the docs site catching up to it. llms.txt rewrites page-tree node URLs before handing them to fumadocs' indexNode, so titles, descriptions, indentation and nesting come out untouched and no pattern runs over an authored description. Node identity is carried by $ref, not url, so the spread leaves title lookup working. The three prose path examples go through localeUrl. llms-full.txt resolves each page's body links against that page's own absolute URL, which also fixes ./sibling and #fragment links that needed the page identity the file never carried. Fenced code blocks are skipped. No new base-URL literal: both routes import SITE_URL and localeUrl from lib/seo.ts, which is unchanged. robots.ts's own BASE collapses into the same constant, taking the count of hostname literals in apps/docs from 4 to 3. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01G5zjYc2BoFV2NjKBBapC7C --- apps/docs/app/llms-full.txt/route.ts | 67 +++++++++++++++++++++++++++- apps/docs/app/llms.txt/route.ts | 59 +++++++++++++++++++++--- apps/docs/app/robots.ts | 7 ++- 3 files changed, 121 insertions(+), 12 deletions(-) diff --git a/apps/docs/app/llms-full.txt/route.ts b/apps/docs/app/llms-full.txt/route.ts index 14f0a50..722f147 100644 --- a/apps/docs/app/llms-full.txt/route.ts +++ b/apps/docs/app/llms-full.txt/route.ts @@ -1,4 +1,5 @@ import { i18n } from '@/lib/i18n'; +import { SITE_URL } from '@/lib/seo'; import { getLLMText, source } from '@/lib/source'; export const revalidate = false; @@ -15,8 +16,72 @@ export const revalidate = false; */ const LANG = i18n.defaultLanguage; +/** Opening or closing line of a fenced code block. */ +const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})/; + +/** + * A markdown link whose target is a page on this site: site-root (`/docs/x`), + * page-relative (`./x`, `../x`), or a bare fragment (`#x`). + * + * Anything carrying a scheme — `https:`, `mailto:` — cannot match, so an + * already-absolute URL is never re-parsed and re-serialized. Nor can a + * protocol-relative `//host/x`, which names a different origin and is not this + * file's business: `\.{0,2}\/` matches the first slash and `(?!\/)` then + * rejects the second. + */ +const SITE_LINK = /(\]\()((?:\.{0,2}\/(?!\/)|#)[^)\s]*)(\))/g; + +/** + * `text`, with every link to a page on this site resolved against `pageUrl`. + * + * Why the fix belongs here and not in `content/docs`: a relative link is + * *correct* on the website, where the origin is attached to the document the + * reader is looking at. It stops being correct the moment this route lifts the + * text out of that document and concatenates it into a file whose whole purpose + * is to be read somewhere else. The defect is a property of this surface, so + * the repair is too — and the authored MDX stays the plain, portable thing it + * should be. + * + * Fenced code blocks are skipped. A code sample containing markdown link syntax + * is documenting that syntax, and rewriting it would be a silent edit to + * authored content. No page has one today — of the 549 on-site links in the + * served body, 0 sit inside a fence — so this is a guard against the page that + * gets written next, not a fix for one that exists. + */ +function absoluteLinks(text: string, pageUrl: string): string { + const lines = text.split('\n'); + let fence: string | undefined; + + for (let i = 0; i < lines.length; i++) { + const marker = FENCE.exec(lines[i])?.[1]; + if (marker) { + if (fence === undefined) fence = marker[0]; + else if (marker[0] === fence) fence = undefined; + continue; + } + if (fence !== undefined) continue; + + lines[i] = lines[i].replace( + SITE_LINK, + (_match, open: string, href: string, close: string) => + `${open}${new URL(href, pageUrl).href}${close}`, + ); + } + + return lines.join('\n'); +} + export async function GET() { - const scan = source.getPages(LANG).map(getLLMText); + // Each page is rewritten against its own URL rather than the joined body + // against the site root: `./objectql` means something different on + // `/docs/reference/cel` than it does three sections away, and a page whose + // text left a code fence unclosed cannot then leak that state into the next + // page's links. + const scan = source + .getPages(LANG) + .map(async (page) => + absoluteLinks(await getLLMText(page), `${SITE_URL}${page.url}`), + ); const scanned = await Promise.all(scan); return new Response(scanned.join('\n\n')); diff --git a/apps/docs/app/llms.txt/route.ts b/apps/docs/app/llms.txt/route.ts index 72fcef0..790d942 100644 --- a/apps/docs/app/llms.txt/route.ts +++ b/apps/docs/app/llms.txt/route.ts @@ -1,6 +1,7 @@ -import type { Folder } from 'fumadocs-core/page-tree'; +import type { Folder, Item, Node } from 'fumadocs-core/page-tree'; import { llms } from 'fumadocs-core/source/llms'; import { i18n } from '@/lib/i18n'; +import { SITE_URL, localeUrl } from '@/lib/seo'; import { source } from '@/lib/source'; export const revalidate = false; @@ -43,6 +44,45 @@ function sectionHeading(folder: Folder): string { return typeof folder.name === 'string' ? folder.name : 'Documentation'; } +/** + * The same page node with an absolute URL. + * + * `indexNode` formats each bullet from `node.url`, and fumadocs fills that + * field with a site-relative path (`/docs/quickstart`). This file exists to be + * fetched and have its text lifted into a context window where the origin is no + * longer attached to it, so a site-relative path resolves only if whatever + * moved it there also carried the base URL — which is precisely what a consumer + * of an `llms.txt` will not do. The marketing site's own `/llms.txt` has always + * emitted absolute URLs; this is the docs site catching up to it. + * + * Rewriting the node rather than the rendered line confines the edit to the + * URL: titles, descriptions, indentation and nesting come out of `indexNode` + * untouched, and no pattern ever runs over an authored description. The node's + * identity is carried by `$ref`, not `url` — `getNodePage` and `getNodeMeta` + * both look up by `$ref` — so the spread leaves title and description lookup + * working. + * + * `SITE_URL` and not `localeUrl` on purpose: `node.url` already carries + * whatever locale prefix the tree was built for, and `localeUrl` would apply a + * second one. `localeUrl` is the right helper for a *logical* path, which is + * how the prose examples below use it. + */ +function absolutePage(page: Item): Item { + return { ...page, url: `${SITE_URL}${page.url}` }; +} + +function absoluteNode(node: Node): Node { + if (node.type === 'page') return absolutePage(node); + if (node.type === 'folder') { + return { + ...node, + index: node.index && absolutePage(node.index), + children: node.children.map(absoluteNode), + }; + } + return node; +} + export async function GET() { const generator = llms(source); const tree = source.getPageTree(LANG); @@ -55,8 +95,9 @@ export async function GET() { 'This is the ObjectOS product and developer documentation, grouped by the ' + 'sections used in the site navigation. Every page below is also available ' + 'as Markdown by appending `.mdx` to its URL (for example ' + - '`/docs/quickstart.mdx`), and `/llms-full.txt` carries the full text of ' + - 'every page in one file.', + `\`${localeUrl(LANG, 'docs/quickstart.mdx')}\`), and ` + + `\`${SITE_URL}/llms-full.txt\` carries the full text of every page in ` + + 'one file.', ]; // Root-level pages (index, why, quickstart, ...) come before the section @@ -65,7 +106,9 @@ export async function GET() { const rootPages = tree.children.filter((node) => node.type === 'page'); if (rootPages.length > 0) { lines.push('', `## ${ROOT_HEADING}`, ''); - for (const node of rootPages) lines.push(generator.indexNode(node, LANG)); + for (const node of rootPages) { + lines.push(generator.indexNode(absolutePage(node), LANG)); + } } for (const node of tree.children) { @@ -74,8 +117,10 @@ export async function GET() { // The folder's own bullet is dropped: the heading already names it. Its // index page and children are rendered at the top level of the section, // so nested subfolders keep exactly one level of indentation. - if (node.index) lines.push(generator.indexNode(node.index, LANG)); - for (const child of node.children) lines.push(generator.indexNode(child, LANG)); + if (node.index) lines.push(generator.indexNode(absolutePage(node.index), LANG)); + for (const child of node.children) { + lines.push(generator.indexNode(absoluteNode(child), LANG)); + } } if (OTHER_LOCALES.length > 0) { @@ -84,7 +129,7 @@ export async function GET() { '## Other Languages', '', `Every page above is also published under a locale prefix — for example ` + - `\`/${OTHER_LOCALES[0]}/docs/quickstart\`. Available locales: ` + + `\`${localeUrl(OTHER_LOCALES[0], 'docs/quickstart')}\`. Available locales: ` + `${OTHER_LOCALES.map((lang) => `\`${lang}\``).join(', ')}. English is the ` + `source of truth; a page with no translation yet falls back to English.`, ); diff --git a/apps/docs/app/robots.ts b/apps/docs/app/robots.ts index b23007e..52e7349 100644 --- a/apps/docs/app/robots.ts +++ b/apps/docs/app/robots.ts @@ -1,6 +1,5 @@ import type { MetadataRoute } from 'next'; - -const BASE = 'https://docs.objectos.ai'; +import { SITE_URL } from '@/lib/seo'; export const revalidate = false; @@ -16,7 +15,7 @@ export default function robots(): MetadataRoute.Robots { // page would. `/api/` stays out: it serves the search index, not readable content. disallow: ['/api/'], }, - sitemap: `${BASE}/sitemap.xml`, - host: BASE, + sitemap: `${SITE_URL}/sitemap.xml`, + host: SITE_URL, }; }