fix(docs): emit absolute URLs from llms.txt and llms-full.txt

apps/docs/app/llms-full.txt/route.ts

@@ -1,4 +1,5 @@
 import { i18n } from '@/lib/i18n';
+import { SITE_URL } from '@/lib/seo';
 import { getLLMText, source } from '@/lib/source';
 
 export const revalidate = false;
@@ -15,8 +16,72 @@ export const revalidate = false;
  */
 const LANG = i18n.defaultLanguage;
 
+/** Opening or closing line of a fenced code block. */
+const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})/;
+
+/**
+ * A markdown link whose target is a page on this site: site-root (`/docs/x`),
+ * page-relative (`./x`, `../x`), or a bare fragment (`#x`).
+ *
+ * Anything carrying a scheme — `https:`, `mailto:` — cannot match, so an
+ * already-absolute URL is never re-parsed and re-serialized. Nor can a
+ * protocol-relative `//host/x`, which names a different origin and is not this
+ * file's business: `\.{0,2}\/` matches the first slash and `(?!\/)` then
+ * rejects the second.
+ */
+const SITE_LINK = /(\]\()((?:\.{0,2}\/(?!\/)|#)[^)\s]*)(\))/g;
+
+/**
+ * `text`, with every link to a page on this site resolved against `pageUrl`.
+ *
+ * Why the fix belongs here and not in `content/docs`: a relative link is
+ * *correct* on the website, where the origin is attached to the document the
+ * reader is looking at. It stops being correct the moment this route lifts the
+ * text out of that document and concatenates it into a file whose whole purpose
+ * is to be read somewhere else. The defect is a property of this surface, so
+ * the repair is too — and the authored MDX stays the plain, portable thing it
+ * should be.
+ *
+ * Fenced code blocks are skipped. A code sample containing markdown link syntax
+ * is documenting that syntax, and rewriting it would be a silent edit to
+ * authored content. No page has one today — of the 549 on-site links in the
+ * served body, 0 sit inside a fence — so this is a guard against the page that
+ * gets written next, not a fix for one that exists.
+ */
+function absoluteLinks(text: string, pageUrl: string): string {
+  const lines = text.split('\n');
+  let fence: string | undefined;
+
+  for (let i = 0; i < lines.length; i++) {
+    const marker = FENCE.exec(lines[i])?.[1];
+    if (marker) {
+      if (fence === undefined) fence = marker[0];
+      else if (marker[0] === fence) fence = undefined;
+      continue;
+    }
+    if (fence !== undefined) continue;
+
+    lines[i] = lines[i].replace(
+      SITE_LINK,
+      (_match, open: string, href: string, close: string) =>
+        `${open}${new URL(href, pageUrl).href}${close}`,
+    );
+  }
+
+  return lines.join('\n');
+}
+
 export async function GET() {
-  const scan = source.getPages(LANG).map(getLLMText);
+  // Each page is rewritten against its own URL rather than the joined body
+  // against the site root: `./objectql` means something different on
+  // `/docs/reference/cel` than it does three sections away, and a page whose
+  // text left a code fence unclosed cannot then leak that state into the next
+  // page's links.
+  const scan = source
+    .getPages(LANG)
+    .map(async (page) =>
+      absoluteLinks(await getLLMText(page), `${SITE_URL}${page.url}`),
+    );
   const scanned = await Promise.all(scan);
 
   return new Response(scanned.join('\n\n'));

apps/docs/app/llms.txt/route.ts

@@ -1,6 +1,7 @@
-import type { Folder } from 'fumadocs-core/page-tree';
+import type { Folder, Item, Node } from 'fumadocs-core/page-tree';
 import { llms } from 'fumadocs-core/source/llms';
 import { i18n } from '@/lib/i18n';
+import { SITE_URL, localeUrl } from '@/lib/seo';
 import { source } from '@/lib/source';
 
 export const revalidate = false;
@@ -43,6 +44,45 @@ function sectionHeading(folder: Folder): string {
   return typeof folder.name === 'string' ? folder.name : 'Documentation';
 }
 
+/**
+ * The same page node with an absolute URL.
+ *
+ * `indexNode` formats each bullet from `node.url`, and fumadocs fills that
+ * field with a site-relative path (`/docs/quickstart`). This file exists to be
+ * fetched and have its text lifted into a context window where the origin is no
+ * longer attached to it, so a site-relative path resolves only if whatever
+ * moved it there also carried the base URL — which is precisely what a consumer
+ * of an `llms.txt` will not do. The marketing site's own `/llms.txt` has always
+ * emitted absolute URLs; this is the docs site catching up to it.
+ *
+ * Rewriting the node rather than the rendered line confines the edit to the
+ * URL: titles, descriptions, indentation and nesting come out of `indexNode`
+ * untouched, and no pattern ever runs over an authored description. The node's
+ * identity is carried by `$ref`, not `url` — `getNodePage` and `getNodeMeta`
+ * both look up by `$ref` — so the spread leaves title and description lookup
+ * working.
+ *
+ * `SITE_URL` and not `localeUrl` on purpose: `node.url` already carries
+ * whatever locale prefix the tree was built for, and `localeUrl` would apply a
+ * second one. `localeUrl` is the right helper for a *logical* path, which is
+ * how the prose examples below use it.
+ */
+function absolutePage(page: Item): Item {
+  return { ...page, url: `${SITE_URL}${page.url}` };
+}
+
+function absoluteNode(node: Node): Node {
+  if (node.type === 'page') return absolutePage(node);
+  if (node.type === 'folder') {
+    return {
+      ...node,
+      index: node.index && absolutePage(node.index),
+      children: node.children.map(absoluteNode),
+    };
+  }
+  return node;
+}
+
 export async function GET() {
   const generator = llms(source);
   const tree = source.getPageTree(LANG);
@@ -55,8 +95,9 @@ export async function GET() {
     'This is the ObjectOS product and developer documentation, grouped by the ' +
       'sections used in the site navigation. Every page below is also available ' +
       'as Markdown by appending `.mdx` to its URL (for example ' +
-      '`/docs/quickstart.mdx`), and `/llms-full.txt` carries the full text of ' +
-      'every page in one file.',
+      `\`${localeUrl(LANG, 'docs/quickstart.mdx')}\`), and ` +
+      `\`${SITE_URL}/llms-full.txt\` carries the full text of every page in ` +
+      'one file.',
   ];
 
   // Root-level pages (index, why, quickstart, ...) come before the section
@@ -65,7 +106,9 @@ export async function GET() {
   const rootPages = tree.children.filter((node) => node.type === 'page');
   if (rootPages.length > 0) {
     lines.push('', `## ${ROOT_HEADING}`, '');
-    for (const node of rootPages) lines.push(generator.indexNode(node, LANG));
+    for (const node of rootPages) {
+      lines.push(generator.indexNode(absolutePage(node), LANG));
+    }
   }
 
   for (const node of tree.children) {
@@ -74,8 +117,10 @@ export async function GET() {
     // The folder's own bullet is dropped: the heading already names it. Its
     // index page and children are rendered at the top level of the section,
     // so nested subfolders keep exactly one level of indentation.
-    if (node.index) lines.push(generator.indexNode(node.index, LANG));
-    for (const child of node.children) lines.push(generator.indexNode(child, LANG));
+    if (node.index) lines.push(generator.indexNode(absolutePage(node.index), LANG));
+    for (const child of node.children) {
+      lines.push(generator.indexNode(absoluteNode(child), LANG));
+    }
   }
 
   if (OTHER_LOCALES.length > 0) {
@@ -84,7 +129,7 @@ export async function GET() {
       '## Other Languages',
       '',
       `Every page above is also published under a locale prefix — for example ` +
-        `\`/${OTHER_LOCALES[0]}/docs/quickstart\`. Available locales: ` +
+        `\`${localeUrl(OTHER_LOCALES[0], 'docs/quickstart')}\`. Available locales: ` +
         `${OTHER_LOCALES.map((lang) => `\`${lang}\``).join(', ')}. English is the ` +
         `source of truth; a page with no translation yet falls back to English.`,
     );

apps/docs/app/robots.ts

@@ -1,6 +1,5 @@
 import type { MetadataRoute } from 'next';
-
-const BASE = 'https://docs.objectos.ai';
+import { SITE_URL } from '@/lib/seo';
 
 export const revalidate = false;
 
@@ -16,7 +15,7 @@ export default function robots(): MetadataRoute.Robots {
       // page would. `/api/` stays out: it serves the search index, not readable content.
       disallow: ['/api/'],
     },
-    sitemap: `${BASE}/sitemap.xml`,
-    host: BASE,
+    sitemap: `${SITE_URL}/sitemap.xml`,
+    host: SITE_URL,
   };
 }