diff --git a/apps/docs/app/llms-full.txt/route.ts b/apps/docs/app/llms-full.txt/route.ts index 14f0a50..722f147 100644 --- a/apps/docs/app/llms-full.txt/route.ts +++ b/apps/docs/app/llms-full.txt/route.ts @@ -1,4 +1,5 @@ import { i18n } from '@/lib/i18n'; +import { SITE_URL } from '@/lib/seo'; import { getLLMText, source } from '@/lib/source'; export const revalidate = false; @@ -15,8 +16,72 @@ export const revalidate = false; */ const LANG = i18n.defaultLanguage; +/** Opening or closing line of a fenced code block. */ +const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})/; + +/** + * A markdown link whose target is a page on this site: site-root (`/docs/x`), + * page-relative (`./x`, `../x`), or a bare fragment (`#x`). + * + * Anything carrying a scheme — `https:`, `mailto:` — cannot match, so an + * already-absolute URL is never re-parsed and re-serialized. Nor can a + * protocol-relative `//host/x`, which names a different origin and is not this + * file's business: `\.{0,2}\/` matches the first slash and `(?!\/)` then + * rejects the second. + */ +const SITE_LINK = /(\]\()((?:\.{0,2}\/(?!\/)|#)[^)\s]*)(\))/g; + +/** + * `text`, with every link to a page on this site resolved against `pageUrl`. + * + * Why the fix belongs here and not in `content/docs`: a relative link is + * *correct* on the website, where the origin is attached to the document the + * reader is looking at. It stops being correct the moment this route lifts the + * text out of that document and concatenates it into a file whose whole purpose + * is to be read somewhere else. The defect is a property of this surface, so + * the repair is too — and the authored MDX stays the plain, portable thing it + * should be. + * + * Fenced code blocks are skipped. A code sample containing markdown link syntax + * is documenting that syntax, and rewriting it would be a silent edit to + * authored content. No page has one today — of the 549 on-site links in the + * served body, 0 sit inside a fence — so this is a guard against the page that + * gets written next, not a fix for one that exists. + */ +function absoluteLinks(text: string, pageUrl: string): string { + const lines = text.split('\n'); + let fence: string | undefined; + + for (let i = 0; i < lines.length; i++) { + const marker = FENCE.exec(lines[i])?.[1]; + if (marker) { + if (fence === undefined) fence = marker[0]; + else if (marker[0] === fence) fence = undefined; + continue; + } + if (fence !== undefined) continue; + + lines[i] = lines[i].replace( + SITE_LINK, + (_match, open: string, href: string, close: string) => + `${open}${new URL(href, pageUrl).href}${close}`, + ); + } + + return lines.join('\n'); +} + export async function GET() { - const scan = source.getPages(LANG).map(getLLMText); + // Each page is rewritten against its own URL rather than the joined body + // against the site root: `./objectql` means something different on + // `/docs/reference/cel` than it does three sections away, and a page whose + // text left a code fence unclosed cannot then leak that state into the next + // page's links. + const scan = source + .getPages(LANG) + .map(async (page) => + absoluteLinks(await getLLMText(page), `${SITE_URL}${page.url}`), + ); const scanned = await Promise.all(scan); return new Response(scanned.join('\n\n')); diff --git a/apps/docs/app/llms.txt/route.ts b/apps/docs/app/llms.txt/route.ts index 72fcef0..790d942 100644 --- a/apps/docs/app/llms.txt/route.ts +++ b/apps/docs/app/llms.txt/route.ts @@ -1,6 +1,7 @@ -import type { Folder } from 'fumadocs-core/page-tree'; +import type { Folder, Item, Node } from 'fumadocs-core/page-tree'; import { llms } from 'fumadocs-core/source/llms'; import { i18n } from '@/lib/i18n'; +import { SITE_URL, localeUrl } from '@/lib/seo'; import { source } from '@/lib/source'; export const revalidate = false; @@ -43,6 +44,45 @@ function sectionHeading(folder: Folder): string { return typeof folder.name === 'string' ? folder.name : 'Documentation'; } +/** + * The same page node with an absolute URL. + * + * `indexNode` formats each bullet from `node.url`, and fumadocs fills that + * field with a site-relative path (`/docs/quickstart`). This file exists to be + * fetched and have its text lifted into a context window where the origin is no + * longer attached to it, so a site-relative path resolves only if whatever + * moved it there also carried the base URL — which is precisely what a consumer + * of an `llms.txt` will not do. The marketing site's own `/llms.txt` has always + * emitted absolute URLs; this is the docs site catching up to it. + * + * Rewriting the node rather than the rendered line confines the edit to the + * URL: titles, descriptions, indentation and nesting come out of `indexNode` + * untouched, and no pattern ever runs over an authored description. The node's + * identity is carried by `$ref`, not `url` — `getNodePage` and `getNodeMeta` + * both look up by `$ref` — so the spread leaves title and description lookup + * working. + * + * `SITE_URL` and not `localeUrl` on purpose: `node.url` already carries + * whatever locale prefix the tree was built for, and `localeUrl` would apply a + * second one. `localeUrl` is the right helper for a *logical* path, which is + * how the prose examples below use it. + */ +function absolutePage(page: Item): Item { + return { ...page, url: `${SITE_URL}${page.url}` }; +} + +function absoluteNode(node: Node): Node { + if (node.type === 'page') return absolutePage(node); + if (node.type === 'folder') { + return { + ...node, + index: node.index && absolutePage(node.index), + children: node.children.map(absoluteNode), + }; + } + return node; +} + export async function GET() { const generator = llms(source); const tree = source.getPageTree(LANG); @@ -55,8 +95,9 @@ export async function GET() { 'This is the ObjectOS product and developer documentation, grouped by the ' + 'sections used in the site navigation. Every page below is also available ' + 'as Markdown by appending `.mdx` to its URL (for example ' + - '`/docs/quickstart.mdx`), and `/llms-full.txt` carries the full text of ' + - 'every page in one file.', + `\`${localeUrl(LANG, 'docs/quickstart.mdx')}\`), and ` + + `\`${SITE_URL}/llms-full.txt\` carries the full text of every page in ` + + 'one file.', ]; // Root-level pages (index, why, quickstart, ...) come before the section @@ -65,7 +106,9 @@ export async function GET() { const rootPages = tree.children.filter((node) => node.type === 'page'); if (rootPages.length > 0) { lines.push('', `## ${ROOT_HEADING}`, ''); - for (const node of rootPages) lines.push(generator.indexNode(node, LANG)); + for (const node of rootPages) { + lines.push(generator.indexNode(absolutePage(node), LANG)); + } } for (const node of tree.children) { @@ -74,8 +117,10 @@ export async function GET() { // The folder's own bullet is dropped: the heading already names it. Its // index page and children are rendered at the top level of the section, // so nested subfolders keep exactly one level of indentation. - if (node.index) lines.push(generator.indexNode(node.index, LANG)); - for (const child of node.children) lines.push(generator.indexNode(child, LANG)); + if (node.index) lines.push(generator.indexNode(absolutePage(node.index), LANG)); + for (const child of node.children) { + lines.push(generator.indexNode(absoluteNode(child), LANG)); + } } if (OTHER_LOCALES.length > 0) { @@ -84,7 +129,7 @@ export async function GET() { '## Other Languages', '', `Every page above is also published under a locale prefix — for example ` + - `\`/${OTHER_LOCALES[0]}/docs/quickstart\`. Available locales: ` + + `\`${localeUrl(OTHER_LOCALES[0], 'docs/quickstart')}\`. Available locales: ` + `${OTHER_LOCALES.map((lang) => `\`${lang}\``).join(', ')}. English is the ` + `source of truth; a page with no translation yet falls back to English.`, ); diff --git a/apps/docs/app/robots.ts b/apps/docs/app/robots.ts index b23007e..52e7349 100644 --- a/apps/docs/app/robots.ts +++ b/apps/docs/app/robots.ts @@ -1,6 +1,5 @@ import type { MetadataRoute } from 'next'; - -const BASE = 'https://docs.objectos.ai'; +import { SITE_URL } from '@/lib/seo'; export const revalidate = false; @@ -16,7 +15,7 @@ export default function robots(): MetadataRoute.Robots { // page would. `/api/` stays out: it serves the search index, not readable content. disallow: ['/api/'], }, - sitemap: `${BASE}/sitemap.xml`, - host: BASE, + sitemap: `${SITE_URL}/sitemap.xml`, + host: SITE_URL, }; }