From 904fa74cd2e2aecca7c231506407edf25abc9646 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 04:42:34 +0000 Subject: [PATCH 1/3] Add a public catalog sitemap and a complete robots.txt. xo.run now ships /sitemap.xml from the shipped docs catalog and a robots.txt with User-agent rules plus a Sitemap line. The live host is https://xo.run; Privacy, Terms, and github.io stay out of the list. Fixes #15 Co-authored-by: Christoffer Hallas --- docs/implementation.md | 1 + www/README.md | 9 ++- www/SITE.md | 10 ++- www/package.json | 3 +- www/public/_headers | 6 ++ www/public/robots.txt | 4 + www/scripts/verify-sitemap.mjs | 135 +++++++++++++++++++++++++++++++++ www/src/docs/site.ts | 73 +++++++++++++++++- www/vite.config.ts | 56 +++++++++++++- 9 files changed, 289 insertions(+), 8 deletions(-) create mode 100644 www/public/robots.txt create mode 100644 www/scripts/verify-sitemap.mjs diff --git a/docs/implementation.md b/docs/implementation.md index 66b3c8ab..a5719537 100644 --- a/docs/implementation.md +++ b/docs/implementation.md @@ -260,6 +260,7 @@ See [`fixtures.md`](fixtures.md) and [`testing.md`](testing.md). | Language pages | Leaders, examples, Result/Option, modules | | Search index | Content records for new pages | | Snippets | Match **current** `syntax.md` | +| Discovery | `/sitemap.xml` of the public catalog; `/robots.txt` on `https://xo.run` | ### 2.14 Tooling ecosystem (as needed) diff --git a/www/README.md b/www/README.md index 51903dff..2b01a7cc 100644 --- a/www/README.md +++ b/www/README.md @@ -24,8 +24,8 @@ just try # wasm + npm --prefix www run dev `npm run dev` starts the local site. `npm run lint`, `npm run format`, `npm run test`, and `npm run build` validate the site before publishing -`www/dist`. The docs-first homepage, primary nav, and Documents catalog live -in `src/docs/site.ts`. +`www/dist`. The docs-first homepage, primary nav, Documents catalog, and +discovery files live in `src/docs/site.ts`. ## Cloudflare Pages @@ -50,6 +50,11 @@ URLs are real pages. Unknown paths still use the `public/404.html` → `/?/path` bounce and `index.html` restore script. Custom domain: `public/CNAME` → `xo.run`. Wasm bindings stay in `public/echo-wasm/`. +`/robots.txt` is a static file. `/sitemap.xml` is emitted at build from the +public catalog in `src/docs/site.ts` and `src/docs/content.ts`. Both use +`https://xo.run`. Privacy and Terms stay out of the sitemap until those pages +exist. + ## Search Docs search uses MiniSearch over content in `src/docs/`. Open the palette from diff --git a/www/SITE.md b/www/SITE.md index 6342d1c5..b2a386e7 100644 --- a/www/SITE.md +++ b/www/SITE.md @@ -206,6 +206,14 @@ playground run then executes the checked MIR and captures `io.print`. Filesystem, net, process, and tasks fail with a playground-host error. Compile and native run stay on `xo` (LLVM). +## Discovery + +`/sitemap.xml` lists the public catalog on `https://xo.run`: home, Install, +Try, and every shipped docs, Book, Echo 2026, and std page. `/robots.txt` +allows crawlers and points at that sitemap. Do not list Privacy, Terms, or +the GitHub Pages host. + ## Out of scope (later) -Richer download tabs, `/e26` URL rename to `/echo-2026`. +Richer download tabs, `/e26` URL rename to `/echo-2026`. Privacy and Terms +pages. diff --git a/www/package.json b/www/package.json index 1bde3d8f..543b0296 100644 --- a/www/package.json +++ b/www/package.json @@ -10,11 +10,12 @@ "format": "oxfmt --check .", "lint": "oxlint . --ignore-pattern public/echo-wasm", "preview": "vite preview", - "test": "npm run test:docs && npm run test:prose && npm run test:std-ref && npm run test:try", + "test": "npm run test:docs && npm run test:prose && npm run test:std-ref && npm run test:try && npm run test:sitemap", "test:docs": "node scripts/verify-docs-pages.mjs", "test:std-ref": "node scripts/verify-std-reference.mjs", "test:prose": "node scripts/verify-prose.mjs", "test:try": "node scripts/verify-try.mjs", + "test:sitemap": "node scripts/verify-sitemap.mjs", "sync:tree-sitter": "node scripts/sync-tree-sitter.mjs", "postinstall": "npm run sync:tree-sitter" }, diff --git a/www/public/_headers b/www/public/_headers index f2bb5c8f..21d9a9ed 100644 --- a/www/public/_headers +++ b/www/public/_headers @@ -5,3 +5,9 @@ /echo-wasm/*.wasm Content-Type: application/wasm Cache-Control: public, max-age=0, must-revalidate + +/sitemap.xml + Content-Type: application/xml; charset=utf-8 + +/robots.txt + Content-Type: text/plain; charset=utf-8 diff --git a/www/public/robots.txt b/www/public/robots.txt new file mode 100644 index 00000000..96b5b483 --- /dev/null +++ b/www/public/robots.txt @@ -0,0 +1,4 @@ +User-agent: * +Allow: / + +Sitemap: https://xo.run/sitemap.xml diff --git a/www/scripts/verify-sitemap.mjs b/www/scripts/verify-sitemap.mjs new file mode 100644 index 00000000..9dad35e4 --- /dev/null +++ b/www/scripts/verify-sitemap.mjs @@ -0,0 +1,135 @@ +/** + * Verifies public discovery files for xo.run: + * - sitemap.xml lists the public catalog on https://xo.run + * - robots.txt has User-agent rules and a Sitemap: line + * - Privacy, Terms, and github.io stay out until those pages exist + * + * Loads src/docs/site.ts and src/docs/content.ts through Vite SSR. + */ +import { readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { createServer } from "vite"; + +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +const server = await createServer({ + root, + logLevel: "error", + server: { middlewareMode: true }, + appType: "custom", +}); + +try { + const site = await server.ssrLoadModule("/src/docs/site.ts"); + const content = await server.ssrLoadModule("/src/docs/content.ts"); + + const { + collectPublicCatalogPaths, + omittedCatalogPaths, + publicCatalogUrl, + publicSiteOrigin, + publicSurfacePaths, + renderRobotsTxt, + renderSitemapXml, + } = site; + const { docsPages } = content; + + const failures = []; + + function fail(message) { + failures.push(message); + } + + if (publicSiteOrigin !== "https://xo.run") { + fail(`publicSiteOrigin must be https://xo.run, got ${publicSiteOrigin}`); + } + + const catalogPaths = collectPublicCatalogPaths(docsPages.map((page) => page.path)); + const sitemap = renderSitemapXml(catalogPaths); + const robots = renderRobotsTxt(); + const committedRobots = readFileSync(path.join(root, "public/robots.txt"), "utf8"); + + if (!sitemap.startsWith("")) { + fail("sitemap.xml must be a real XML document"); + } + if (!sitemap.includes("")) { + fail("sitemap.xml must use the sitemaps.org urlset namespace"); + } + + const requiredPaths = [ + ...publicSurfacePaths, + "/docs", + "/docs/std", + "/docs/leaders", + "/book", + "/e26", + "/e26/spec", + "/try", + "/install", + ]; + for (const required of requiredPaths) { + if (!catalogPaths.includes(required)) { + fail(`public catalog missing ${required}`); + } + const loc = publicCatalogUrl(required); + if (!sitemap.includes(`${loc}`)) { + fail(`sitemap.xml missing ${loc}`); + } + } + + for (const page of docsPages) { + if (!catalogPaths.includes(page.path)) { + fail(`docs page ${page.path} missing from the public catalog`); + } + } + + for (const omitted of omittedCatalogPaths) { + if (catalogPaths.includes(omitted)) { + fail(`public catalog must not list ${omitted} until that page exists`); + } + if (sitemap.toLowerCase().includes(omitted)) { + fail(`sitemap.xml must not list ${omitted}`); + } + } + + if (/github\.io/i.test(sitemap) || /github\.io/i.test(robots)) { + fail("discovery files must not list the GitHub Pages host"); + } + if (!/https:\/\/xo\.run\//.test(sitemap) || /http:\/\/xo\.run/.test(sitemap)) { + fail("sitemap.xml must use https://xo.run as the live host"); + } + + if (!/^User-agent:\s+\*/m.test(robots)) { + fail("robots.txt must include a User-agent rule"); + } + if (!/^Allow:\s+\//m.test(robots)) { + fail("robots.txt must include an Allow rule"); + } + if (!/^Sitemap:\s+https:\/\/xo\.run\/sitemap\.xml$/m.test(robots)) { + fail("robots.txt must point Sitemap: at https://xo.run/sitemap.xml"); + } + if (committedRobots !== robots) { + fail("public/robots.txt must match renderRobotsTxt()"); + } + + if (failures.length) { + console.error(JSON.stringify({ ok: false, failures }, null, 2)); + process.exitCode = 1; + } else { + console.log( + JSON.stringify( + { + ok: true, + origin: publicSiteOrigin, + urls: catalogPaths.length, + sitemapBytes: sitemap.length, + }, + null, + 2, + ), + ); + } +} finally { + await server.close(); +} diff --git a/www/src/docs/site.ts b/www/src/docs/site.ts index 9858ecf1..05df90e3 100644 --- a/www/src/docs/site.ts +++ b/www/src/docs/site.ts @@ -1,8 +1,21 @@ /** - * Public site chrome: homepage, primary nav, and the Documents hub catalog. - * Pages and tests load this module; do not duplicate the lists in UI or fixtures. + * Public site chrome: homepage, primary nav, Documents hub catalog, and the + * discovery files (`/sitemap.xml`, `/robots.txt`). Pages and tests load this + * module; do not duplicate the lists in UI or fixtures. */ +/** Live public host. Do not emit github.io URLs in discovery files. */ +export const publicSiteOrigin = "https://xo.run"; + +/** Top-level routes that are not docs pages. */ +export const publicSurfacePaths = ["/", "/install", "/try"] as const; + +/** + * Legal routes stay out of the sitemap until the pages exist. + * Do not add these paths here when inventing placeholder URLs. + */ +export const omittedCatalogPaths = ["/privacy", "/terms"] as const; + export type SiteNavItem = { label: string; to: string; @@ -279,6 +292,62 @@ export const docsHubCatalog: DocsCatalogGroup[] = [ }, ]; +/** + * Public catalog paths for `/sitemap.xml`. + * `existingPagePaths` is the shipped HTML catalog (`staticPages` / `docsPages`). + * Privacy and Terms are included only when those pages already exist. + */ +export function collectPublicCatalogPaths(existingPagePaths: readonly string[]): string[] { + const existing = new Set(existingPagePaths); + const paths = new Set([ + ...publicSurfacePaths, + ...publicChromePaths(), + ...existingPagePaths, + ]); + + for (const omitted of omittedCatalogPaths) { + if (!existing.has(omitted)) { + paths.delete(omitted); + } + } + + return [...paths].sort((left, right) => left.localeCompare(right)); +} + +export function publicCatalogUrl(path: string): string { + if (!path.startsWith("/")) { + throw new Error(`catalog path must be absolute, got ${path}`); + } + if (path === "/") { + return `${publicSiteOrigin}/`; + } + return `${publicSiteOrigin}${path}`; +} + +export function renderSitemapXml(paths: readonly string[]): string { + const urls = paths + .map((path) => ` \n ${escapeHtml(publicCatalogUrl(path))}\n `) + .join("\n"); + + return [ + ``, + ``, + urls, + ``, + ``, + ].join("\n"); +} + +export function renderRobotsTxt(): string { + return [ + "User-agent: *", + "Allow: /", + "", + `Sitemap: ${publicSiteOrigin}/sitemap.xml`, + "", + ].join("\n"); +} + function escapeHtml(text: string): string { return text .replaceAll("&", "&") diff --git a/www/vite.config.ts b/www/vite.config.ts index 0806831a..2002fa82 100644 --- a/www/vite.config.ts +++ b/www/vite.config.ts @@ -10,7 +10,12 @@ import { type DocsSearchAsset, type DocsSemanticAsset, } from "./src/docs/search"; -import { renderStaticHomeAndHub } from "./src/docs/site"; +import { + collectPublicCatalogPaths, + renderRobotsTxt, + renderSitemapXml, + renderStaticHomeAndHub, +} from "./src/docs/site"; import { distFileForPath, escapeHtml, staticPages, type StaticPage } from "./src/docs/static-html"; const docsSearchIndexDevFileName = "indices/search.json"; @@ -232,6 +237,47 @@ function applyStaticPage(html: string, page: StaticPage): string { }); } +function publicCatalogPaths() { + return collectPublicCatalogPaths(staticPages().map((page) => page.path)); +} + +function siteDiscoveryPlugin(): Plugin { + return { + name: "site-discovery", + configureServer(server) { + server.middlewares.use((request, response, next) => { + const requestPath = request.url?.split("?", 1)[0] ?? ""; + + if (requestPath === "/sitemap.xml") { + response.setHeader("Content-Type", "application/xml; charset=utf-8"); + response.end(renderSitemapXml(publicCatalogPaths())); + return; + } + + if (requestPath === "/robots.txt") { + response.setHeader("Content-Type", "text/plain; charset=utf-8"); + response.end(renderRobotsTxt()); + return; + } + + next(); + }); + }, + generateBundle() { + this.emitFile({ + type: "asset", + fileName: "sitemap.xml", + source: renderSitemapXml(publicCatalogPaths()), + }); + this.emitFile({ + type: "asset", + fileName: "robots.txt", + source: renderRobotsTxt(), + }); + }, + }; +} + function docsFirstStaticPlugin(): Plugin { const fallbackMarker = ''; @@ -310,5 +356,11 @@ export default defineConfig({ }, }, }, - plugins: [docsSearchIndexPlugin(), docsFirstStaticPlugin(), react(), tailwindcss()], + plugins: [ + siteDiscoveryPlugin(), + docsSearchIndexPlugin(), + docsFirstStaticPlugin(), + react(), + tailwindcss(), + ], }); From 8d57931a16574f56ad052a0e96f7ad4ad1e499a0 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 04:43:27 +0000 Subject: [PATCH 2/3] Format sitemap helpers to match oxfmt. Co-authored-by: Christoffer Hallas --- www/scripts/verify-sitemap.mjs | 4 ++-- www/src/docs/site.ts | 10 +++------- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/www/scripts/verify-sitemap.mjs b/www/scripts/verify-sitemap.mjs index 9dad35e4..f0906b4a 100644 --- a/www/scripts/verify-sitemap.mjs +++ b/www/scripts/verify-sitemap.mjs @@ -50,10 +50,10 @@ try { const robots = renderRobotsTxt(); const committedRobots = readFileSync(path.join(root, "public/robots.txt"), "utf8"); - if (!sitemap.startsWith("")) { + if (!sitemap.startsWith('')) { fail("sitemap.xml must be a real XML document"); } - if (!sitemap.includes("")) { + if (!sitemap.includes('')) { fail("sitemap.xml must use the sitemaps.org urlset namespace"); } diff --git a/www/src/docs/site.ts b/www/src/docs/site.ts index 05df90e3..2dc0340c 100644 --- a/www/src/docs/site.ts +++ b/www/src/docs/site.ts @@ -339,13 +339,9 @@ export function renderSitemapXml(paths: readonly string[]): string { } export function renderRobotsTxt(): string { - return [ - "User-agent: *", - "Allow: /", - "", - `Sitemap: ${publicSiteOrigin}/sitemap.xml`, - "", - ].join("\n"); + return ["User-agent: *", "Allow: /", "", `Sitemap: ${publicSiteOrigin}/sitemap.xml`, ""].join( + "\n", + ); } function escapeHtml(text: string): string { From 841c163174061336e2c4553b60c7e52ccc76d48c Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sun, 16 Aug 2026 05:51:52 +0000 Subject: [PATCH 3/3] List Privacy and Terms in the sitemap only when those pages exist. The catalog now comes from the shipped static HTML pages and site chrome on main. github.io stays out of the discovery files. Co-authored-by: Christoffer Hallas --- www/README.md | 5 ++--- www/SITE.md | 10 +++++----- www/scripts/verify-sitemap.mjs | 31 +++++++++++++++++++++++-------- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/www/README.md b/www/README.md index 2b01a7cc..f9be3cdb 100644 --- a/www/README.md +++ b/www/README.md @@ -51,9 +51,8 @@ bounce and `index.html` restore script. Custom domain: `public/CNAME` → `xo.run`. Wasm bindings stay in `public/echo-wasm/`. `/robots.txt` is a static file. `/sitemap.xml` is emitted at build from the -public catalog in `src/docs/site.ts` and `src/docs/content.ts`. Both use -`https://xo.run`. Privacy and Terms stay out of the sitemap until those pages -exist. +public catalog (`staticPages` and site chrome). Both use `https://xo.run`. +Privacy and Terms are listed only when those pages exist. ## Search diff --git a/www/SITE.md b/www/SITE.md index b2a386e7..6960bc8e 100644 --- a/www/SITE.md +++ b/www/SITE.md @@ -209,11 +209,11 @@ Compile and native run stay on `xo` (LLVM). ## Discovery `/sitemap.xml` lists the public catalog on `https://xo.run`: home, Install, -Try, and every shipped docs, Book, Echo 2026, and std page. `/robots.txt` -allows crawlers and points at that sitemap. Do not list Privacy, Terms, or -the GitHub Pages host. +Try, catalog and footer routes, and every shipped docs, Book, Echo 2026, and +std page. `/robots.txt` allows crawlers and points at that sitemap. Privacy +and Terms are listed only when those pages exist. Do not list the GitHub +Pages host. ## Out of scope (later) -Richer download tabs, `/e26` URL rename to `/echo-2026`. Privacy and Terms -pages. +Richer download tabs, `/e26` URL rename to `/echo-2026`. diff --git a/www/scripts/verify-sitemap.mjs b/www/scripts/verify-sitemap.mjs index f0906b4a..87cd106a 100644 --- a/www/scripts/verify-sitemap.mjs +++ b/www/scripts/verify-sitemap.mjs @@ -2,9 +2,10 @@ * Verifies public discovery files for xo.run: * - sitemap.xml lists the public catalog on https://xo.run * - robots.txt has User-agent rules and a Sitemap: line - * - Privacy, Terms, and github.io stay out until those pages exist + * - Privacy and Terms are listed only when those pages exist + * - github.io is never listed * - * Loads src/docs/site.ts and src/docs/content.ts through Vite SSR. + * Loads src/docs/site.ts, content.ts, and static-html.ts through Vite SSR. */ import { readFileSync } from "node:fs"; import path from "node:path"; @@ -23,17 +24,20 @@ const server = await createServer({ try { const site = await server.ssrLoadModule("/src/docs/site.ts"); const content = await server.ssrLoadModule("/src/docs/content.ts"); + const staticHtml = await server.ssrLoadModule("/src/docs/static-html.ts"); const { collectPublicCatalogPaths, omittedCatalogPaths, publicCatalogUrl, + publicChromePaths, publicSiteOrigin, publicSurfacePaths, renderRobotsTxt, renderSitemapXml, } = site; const { docsPages } = content; + const { staticPages } = staticHtml; const failures = []; @@ -45,7 +49,11 @@ try { fail(`publicSiteOrigin must be https://xo.run, got ${publicSiteOrigin}`); } - const catalogPaths = collectPublicCatalogPaths(docsPages.map((page) => page.path)); + const existingPagePaths = [ + ...new Set([...staticPages().map((page) => page.path), ...docsPages.map((page) => page.path)]), + ]; + const existing = new Set(existingPagePaths); + const catalogPaths = collectPublicCatalogPaths(existingPagePaths); const sitemap = renderSitemapXml(catalogPaths); const robots = renderRobotsTxt(); const committedRobots = readFileSync(path.join(root, "public/robots.txt"), "utf8"); @@ -59,6 +67,7 @@ try { const requiredPaths = [ ...publicSurfacePaths, + ...publicChromePaths(), "/docs", "/docs/std", "/docs/leaders", @@ -84,12 +93,18 @@ try { } } - for (const omitted of omittedCatalogPaths) { - if (catalogPaths.includes(omitted)) { - fail(`public catalog must not list ${omitted} until that page exists`); + for (const candidate of omittedCatalogPaths) { + if (existing.has(candidate)) { + if (!catalogPaths.includes(candidate)) { + fail(`public catalog missing existing page ${candidate}`); + } + continue; + } + if (catalogPaths.includes(candidate)) { + fail(`public catalog must not invent ${candidate}`); } - if (sitemap.toLowerCase().includes(omitted)) { - fail(`sitemap.xml must not list ${omitted}`); + if (sitemap.toLowerCase().includes(candidate)) { + fail(`sitemap.xml must not list ${candidate} until that page exists`); } }