From 88fef72b6f6e0a43d2d160821dee6251d8c3df78 Mon Sep 17 00:00:00 2001 From: MattIPv4 Date: Sun, 23 Aug 2026 14:18:50 +0100 Subject: [PATCH 1/5] Implement sitemap route --- package-lock.json | 66 +++++++++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/routes/index.ts | 49 ++++++++++++++++++++++++++++++++- 3 files changed, 115 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 9ee9e65..59a5ab2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@sentry/cloudflare": "^10.69.0", "@tanstack/react-virtual": "^3.14.9", "algoliasearch": "^5.56.0", + "fast-xml-builder": "^1.3.1", "hono": "^4.12.34", "is-deflate": "^1.0.0", "is-gzip": "^2.0.0", @@ -4544,6 +4545,22 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, "node_modules/fault": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", @@ -5934,6 +5951,21 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -7446,6 +7478,21 @@ "repeat-string": "^1.5.2" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", @@ -10475,6 +10522,15 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "requires": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, "fault": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", @@ -11294,6 +11350,11 @@ "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true }, + "path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==" + }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -12240,6 +12301,11 @@ "repeat-string": "^1.5.2" } }, + "xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==" + }, "yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", diff --git a/package.json b/package.json index 6542d17..1051289 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@sentry/cloudflare": "^10.69.0", "@tanstack/react-virtual": "^3.14.9", "algoliasearch": "^5.56.0", + "fast-xml-builder": "^1.3.1", "hono": "^4.12.34", "is-deflate": "^1.0.0", "is-gzip": "^2.0.0", diff --git a/src/routes/index.ts b/src/routes/index.ts index 9d3488d..647707f 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,11 +1,13 @@ import type { OpenAPIRegistry } from '@asteasolutions/zod-to-openapi'; +import XMLBuilder from 'fast-xml-builder'; import type { Context, Hono } from 'hono'; import bannerPng from '../assets/banner.png'; import faviconIco from '../assets/favicon.ico'; import faviconPng from '../assets/favicon.png'; import faviconSvg from '../assets/favicon.svg'; -import respond, { isWebsite, withCache } from '../utils/respond.ts'; +import { libraries } from '../utils/metadata.ts'; +import respond, { isWebsite, notFound, withCache } from '../utils/respond.ts'; import IndexPage from './index.page.tsx'; @@ -88,6 +90,48 @@ const handleGetOpenSearchXml = (ctx: Context) => { ); }; +/** + * Handle GET /sitemap.xml requests. + * + * @param ctx Request context. + */ +const handleGetSitemapXml = async (ctx: Context) => { + // Only serve a sitemap for website requests + if (!isWebsite(ctx)) { + return notFound(ctx, 'Endpoint'); + } + + // Set a 6 hour life on this response + withCache(ctx, 6 * 60 * 60); + + const origin = new URL(ctx.req.url).origin; + const libraryPaths = (await libraries()).map( + (name) => `/libraries/${encodeURIComponent(name)}`, + ); + const sitemap = new XMLBuilder({ + format: true, + ignoreAttributes: false, + suppressEmptyNode: true, + }).build({ + '?xml': { + '@_version': '1.0', + '@_encoding': 'UTF-8', + }, + urlset: { + '@_xmlns': 'http://www.sitemaps.org/schemas/sitemap/0.9', + url: ['/', '/about', '/api', '/libraries', ...libraryPaths].map( + (path) => ({ + loc: `${origin}${path}`, + }), + ), + }, + }); + + return ctx.body(sitemap, 200, { + 'Content-Type': 'application/xml', + }); +}; + /** * Handle GET /favicon.ico requests. * @@ -172,6 +216,9 @@ export default (app: Hono, _registry: OpenAPIRegistry) => { // Provide OpenSearch support for the website app.get('/opensearch.xml', handleGetOpenSearchXml); + // Provide a sitemap for the website + app.get('/sitemap.xml', handleGetSitemapXml); + // Serve the favicon assets app.get('/favicon.ico', handleGetFaviconIco); app.get('/favicon.png', handleGetFaviconPng); From 8ab13c990792ad587e09fac619de47c65d04b308 Mon Sep 17 00:00:00 2001 From: MattIPv4 Date: Sun, 23 Aug 2026 14:22:19 +0100 Subject: [PATCH 2/5] Add specs for sitemap route --- package-lock.json | 121 +++++++++++++++++++++++++++++++ package.json | 1 + src/routes/index.browser.spec.ts | 46 ++++++++++++ src/routes/index.spec.ts | 22 ++++++ 4 files changed, 190 insertions(+) diff --git a/package-lock.json b/package-lock.json index 59a5ab2..73c225c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,6 +42,7 @@ "eslint": "^10.8.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-jsdoc": "^63.3.2", + "fast-xml-parser": "^5.11.0", "husky": "^9.1.7", "lint-staged": "^17.2.0", "prettier": "^3.9.6", @@ -1866,6 +1867,19 @@ "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.3" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@opentelemetry/api": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", @@ -3610,6 +3624,19 @@ "node": ">= 14.0.0" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/apg-lite": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/apg-lite/-/apg-lite-1.0.5.tgz", @@ -4561,6 +4588,30 @@ "xml-naming": "^0.3.0" } }, + "node_modules/fast-xml-parser": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.0.tgz", + "integrity": "sha512-9IGxMqvqLOnqP+Egi1nqDHKv5k8aZ7r9n558enxcucmyVGEBNPAU+MOg/8jPIS7rO7sSq4gFm1/nHtiaubMruw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fault": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", @@ -5124,6 +5175,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -6746,6 +6810,22 @@ "node": ">=0.6.19" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/stylis": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", @@ -8585,6 +8665,12 @@ "@tybys/wasm-util": "^0.10.3" } }, + "@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "dev": true + }, "@opentelemetry/api": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", @@ -9894,6 +9980,12 @@ "@algolia/requester-node-http": "5.56.0" } }, + "anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "dev": true + }, "apg-lite": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/apg-lite/-/apg-lite-1.0.5.tgz", @@ -10531,6 +10623,20 @@ "xml-naming": "^0.3.0" } }, + "fast-xml-parser": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.0.tgz", + "integrity": "sha512-9IGxMqvqLOnqP+Egi1nqDHKv5k8aZ7r9n558enxcucmyVGEBNPAU+MOg/8jPIS7rO7sSq4gFm1/nHtiaubMruw==", + "dev": true, + "requires": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + } + }, "fault": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", @@ -10862,6 +10968,12 @@ "which-typed-array": "^1.1.16" } }, + "is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "dev": true + }, "isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -11855,6 +11967,15 @@ "integrity": "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==", "dev": true }, + "strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "dev": true, + "requires": { + "anynum": "^1.0.1" + } + }, "stylis": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", diff --git a/package.json b/package.json index 1051289..2de7cf1 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "eslint": "^10.8.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-jsdoc": "^63.3.2", + "fast-xml-parser": "^5.11.0", "husky": "^9.1.7", "lint-staged": "^17.2.0", "prettier": "^3.9.6", diff --git a/src/routes/index.browser.spec.ts b/src/routes/index.browser.spec.ts index ef74ec5..cc46b7b 100644 --- a/src/routes/index.browser.spec.ts +++ b/src/routes/index.browser.spec.ts @@ -1,5 +1,14 @@ +import { XMLParser } from 'fast-xml-parser'; + import { expect, test } from '../utils/spec/playwright.ts'; +interface Sitemap { + urlset?: { + '@_xmlns'?: string; + url?: { loc?: string } | { loc?: string }[]; + }; +} + test.describe('/', () => { test('renders page', async ({ page }) => { const response = await page.goto('/'); @@ -96,6 +105,43 @@ test.describe('/opensearch.xml', () => { }); }); +test.describe('/sitemap.xml', () => { + test('valid response', async ({ page }) => { + const response = await page.request.get('/sitemap.xml'); + expect(response?.ok()).toBe(true); + expect(response?.status()).toBe(200); + expect(response?.headers()['cache-control']).toBe( + 'public, max-age=21600', + ); // 6 hours + expect(response?.headers()['content-type']).toMatch( + /^application\/xml(;|$)/, + ); + + const body = await response.text(); + const origin = new URL(response.url()).origin; + const sitemap = new XMLParser({ + ignoreAttributes: false, + }).parse(body) as unknown as Sitemap; + const urls = sitemap.urlset?.url; + + expect(sitemap.urlset?.['@_xmlns']).toBe( + 'http://www.sitemaps.org/schemas/sitemap/0.9', + ); + expect(Array.isArray(urls)).toBe(true); + if (!Array.isArray(urls)) throw new Error('Missing sitemap URLs'); + + const locations = urls.map((url) => url.loc); + expect(locations).toContain(`${origin}/`); + expect(locations).toContain(`${origin}/about`); + expect(locations).toContain(`${origin}/api`); + expect(locations).toContain(`${origin}/libraries`); + expect(locations).toContain(`${origin}/libraries/backbone.js`); + expect(locations).not.toContain( + `${origin}/libraries/backbone.js/1.1.0`, + ); + }); +}); + test.describe('/favicon.ico', () => { test('valid response', async ({ page }) => { const response = await page.request.get('/favicon.ico'); diff --git a/src/routes/index.spec.ts b/src/routes/index.spec.ts index c2e7764..d76850c 100644 --- a/src/routes/index.spec.ts +++ b/src/routes/index.spec.ts @@ -115,6 +115,28 @@ describe('/opensearch.xml', () => { }); }); +describe('/sitemap.xml', () => { + const path = '/sitemap.xml'; + + describe('No query params', () => { + const response = beforeRequest(path); + + testCors(path, response); + it('returns the correct status code', () => { + expect(response.status).to.eq(404); + }); + it('does not return a sitemap', async () => { + expect(response.headers.get('Content-Type')).to.match( + /application\/json/, + ); + expect(await response.json()).to.have.property( + 'message', + 'Endpoint not found', + ); + }); + }); +}); + describe('/favicon.ico', () => { // Fetch the endpoint const path = '/favicon.ico'; From 3640a261b382dee5f771967419727361ea0ab4a6 Mon Sep 17 00:00:00 2001 From: MattIPv4 Date: Sun, 23 Aug 2026 14:25:01 +0100 Subject: [PATCH 3/5] Use fast-xml-* for opensearch route --- src/routes/index.browser.spec.ts | 34 ++++++++++++++++-- src/routes/index.spec.ts | 16 ++++----- src/routes/index.ts | 61 +++++++++++++++++++++++--------- 3 files changed, 83 insertions(+), 28 deletions(-) diff --git a/src/routes/index.browser.spec.ts b/src/routes/index.browser.spec.ts index cc46b7b..06cee4b 100644 --- a/src/routes/index.browser.spec.ts +++ b/src/routes/index.browser.spec.ts @@ -9,6 +9,19 @@ interface Sitemap { }; } +interface OpenSearch { + OpenSearchDescription?: { + '@_xmlns'?: string; + '@_xmlns:moz'?: string; + ShortName?: string; + Url?: { + '@_type'?: string; + '@_method'?: string; + '@_template'?: string; + }; + }; +} + test.describe('/', () => { test('renders page', async ({ page }) => { const response = await page.goto('/'); @@ -99,8 +112,25 @@ test.describe('/opensearch.xml', () => { expect(response?.headers()['content-type']).toMatch( /^application\/opensearchdescription\+xml(;|$)/, ); - expect(await response?.text()).toContain( - '/libraries?search={searchTerms}', + + const body = await response.text(); + const origin = new URL(response.url()).origin; + const opensearch = new XMLParser({ + ignoreAttributes: false, + }).parse(body) as unknown as OpenSearch; + const description = opensearch.OpenSearchDescription; + + expect(description?.['@_xmlns']).toBe( + 'http://a9.com/-/spec/opensearch/1.1/', + ); + expect(description?.['@_xmlns:moz']).toBe( + 'http://www.mozilla.org/2006/browser/search/', + ); + expect(description?.ShortName).toBe('cdnjs'); + expect(description?.Url?.['@_type']).toBe('text/html'); + expect(description?.Url?.['@_method']).toBe('GET'); + expect(description?.Url?.['@_template']).toBe( + `${origin}/libraries?search={searchTerms}`, ); }); }); diff --git a/src/routes/index.spec.ts b/src/routes/index.spec.ts index d76850c..08a63af 100644 --- a/src/routes/index.spec.ts +++ b/src/routes/index.spec.ts @@ -96,20 +96,16 @@ describe('/opensearch.xml', () => { // Test the endpoint testCors(path, response); - it('returns the correct Cache headers', () => { - expect(response.headers.get('Cache-Control')).to.eq( - 'public, max-age=30672000, immutable', - ); // 355 days - }); it('returns the correct status code', () => { - expect(response.status).to.eq(200); + expect(response.status).to.eq(404); }); - it('includes the libraries endpoint for searching', async () => { + it('does not return an OpenSearch description', async () => { expect(response.headers.get('Content-Type')).to.match( - /application\/opensearchdescription\+xml/, + /application\/json/, ); - expect(await response.text()).to.include( - `/libraries?search={searchTerms}`, + expect(await response.json()).to.have.property( + 'message', + 'Endpoint not found', ); }); }); diff --git a/src/routes/index.ts b/src/routes/index.ts index 647707f..b10658e 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -66,28 +66,57 @@ const handleGetRobotsTxt = (ctx: Context) => { * @param ctx Request context. */ const handleGetOpenSearchXml = (ctx: Context) => { + // Only serve OpenSearch for website requests + if (!isWebsite(ctx)) { + return notFound(ctx, 'Endpoint'); + } + // Set a 355 day (same as CDN) life on this response // This is also immutable withCache(ctx, 355 * 24 * 60 * 60, true); - // Respond const origin = new URL(ctx.req.url).origin; - return ctx.body( - ` - - cdnjs - cdnjs is the free, open-source CDN for the web's most popular libraries. JavaScript, CSS, and font resources, globally cached on Cloudflare's network. Trusted by 12.5% of all websites, serving 250 billion requests per month. - UTF-8 - ${origin}/favicon.ico - ${origin}/favicon.png - ${origin}/favicon.svg - -`, - 200, - { - 'Content-Type': 'application/opensearchdescription+xml', + const opensearch = new XMLBuilder({ + format: true, + ignoreAttributes: false, + suppressEmptyNode: true, + }).build({ + '?xml': { + '@_version': '1.0', + '@_encoding': 'UTF-8', }, - ); + OpenSearchDescription: { + '@_xmlns': 'http://a9.com/-/spec/opensearch/1.1/', + '@_xmlns:moz': 'http://www.mozilla.org/2006/browser/search/', + ShortName: 'cdnjs', + Description: + "cdnjs is the free, open-source CDN for the web's most popular libraries. JavaScript, CSS, and font resources, globally cached on Cloudflare's network. Trusted by 12.5% of all websites, serving 250 billion requests per month.", + InputEncoding: 'UTF-8', + Image: [ + { + '@_type': 'image/x-icon', + '#text': `${origin}/favicon.ico`, + }, + { + '@_type': 'image/png', + '#text': `${origin}/favicon.png`, + }, + { + '@_type': 'image/svg+xml', + '#text': `${origin}/favicon.svg`, + }, + ], + Url: { + '@_type': 'text/html', + '@_method': 'GET', + '@_template': `${origin}/libraries?search={searchTerms}`, + }, + }, + }); + + return ctx.body(opensearch, 200, { + 'Content-Type': 'application/opensearchdescription+xml', + }); }; /** From 7733724344f23241139c146fca8dd03b1d4c8d03 Mon Sep 17 00:00:00 2001 From: MattIPv4 Date: Sun, 23 Aug 2026 15:56:47 +0100 Subject: [PATCH 4/5] Return indexable robots.txt for production --- src/routes/index.browser.spec.ts | 58 ++++++++++++++++++++++++++++++-- src/routes/index.ts | 8 +++++ src/utils/spec/playwright.ts | 19 ++++++----- 3 files changed, 74 insertions(+), 11 deletions(-) diff --git a/src/routes/index.browser.spec.ts b/src/routes/index.browser.spec.ts index 06cee4b..b49a2ed 100644 --- a/src/routes/index.browser.spec.ts +++ b/src/routes/index.browser.spec.ts @@ -1,6 +1,6 @@ import { XMLParser } from 'fast-xml-parser'; -import { expect, test } from '../utils/spec/playwright.ts'; +import { createServer, expect, test } from '../utils/spec/playwright.ts'; interface Sitemap { urlset?: { @@ -87,8 +87,17 @@ test.describe('/health', () => { }); test.describe('/robots.txt', () => { - test('valid response', async ({ page }) => { - const response = await page.request.get('/robots.txt'); + const isProduction = + process.env.PLAYWRIGHT_EXTERNAL_WEB_URL?.replace(/\/+$/, '') === + 'https://cdnjs.com'; + + test('blocks indexing on non-production websites', async ({ page }) => { + test.skip( + isProduction, + 'Production website robots.txt does not block indexing', + ); + + const response = await page.goto('/robots.txt'); expect(response?.ok()).toBe(true); expect(response?.status()).toBe(200); expect(response?.headers()['cache-control']).toBe( @@ -99,6 +108,49 @@ test.describe('/robots.txt', () => { ); expect(await response?.text()).toBe('User-agent: *\nDisallow: /'); }); + + test('allows indexing on production website', async ({ page }) => { + // If we're not running against production, map the /robots.txt request + // to a test server that simulates the production environment + let cleanup = () => Promise.resolve(); + if (!isProduction) { + const server = createServer({ WEBSITE_BASE: 'https://cdnjs.com' }); + await server.listen(); + + const route = await page.route('/robots.txt', async (route) => { + const response = await server + .getWorker() + .fetch('https://cdnjs.com/robots.txt'); + route.fulfill({ + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + body: await response.text(), + }); + }); + + cleanup = async () => { + await route.dispose(); + await server.close(); + }; + } + + try { + const response = await page.goto('/robots.txt'); + expect(response?.ok()).toBe(true); + expect(response?.status()).toBe(200); + expect(response?.headers()['cache-control']).toBe( + 'public, max-age=30672000, immutable', + ); // 355 days + expect(response?.headers()['content-type']).toMatch( + /^text\/plain(;|$)/, + ); + expect(await response?.text()).toBe( + 'User-agent: *\nAllow: /\nSitemap: https://cdnjs.com/sitemap.xml', + ); + } finally { + await cleanup(); + } + }); }); test.describe('/opensearch.xml', () => { diff --git a/src/routes/index.ts b/src/routes/index.ts index b10658e..7d32231 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,4 +1,5 @@ import type { OpenAPIRegistry } from '@asteasolutions/zod-to-openapi'; +import { env } from 'cloudflare:workers'; import XMLBuilder from 'fast-xml-builder'; import type { Context, Hono } from 'hono'; @@ -56,6 +57,13 @@ const handleGetRobotsTxt = (ctx: Context) => { // This is also immutable withCache(ctx, 355 * 24 * 60 * 60, true); + // Only the production site at cdnjs.com should ever be indexable + if (env.WEBSITE_BASE === 'https://cdnjs.com' && isWebsite(ctx)) { + return ctx.text( + 'User-agent: *\nAllow: /\nSitemap: https://cdnjs.com/sitemap.xml', + ); + } + // Disallow all robots return ctx.text('User-agent: *\nDisallow: /'); }; diff --git a/src/utils/spec/playwright.ts b/src/utils/spec/playwright.ts index 30c1051..6aaa802 100644 --- a/src/utils/spec/playwright.ts +++ b/src/utils/spec/playwright.ts @@ -14,6 +14,16 @@ interface WorkerFixtures { server?: TestHarness; } +export const createServer = (vars?: Record) => + createTestHarness({ + workers: [ + { + configPath, + vars: { WEBSITE_BASE: websiteBase, ...vars }, + }, + ], + }); + export const test = base.extend({ server: [ async ({ browserName }, use) => { @@ -28,14 +38,7 @@ export const test = base.extend({ return; } - const server = createTestHarness({ - workers: [ - { - configPath, - vars: { WEBSITE_BASE: websiteBase }, - }, - ], - }); + const server = createServer(); try { await use(server); From dd2e0bde38fd47b6f907a9fa6adab84b54378b64 Mon Sep 17 00:00:00 2001 From: MattIPv4 Date: Mon, 24 Aug 2026 13:38:20 +0100 Subject: [PATCH 5/5] Compare origin against WEBSITE_BASE --- src/utils/respond.ts | 15 +++++++++++++-- src/utils/spec/playwright.ts | 10 +++++----- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/utils/respond.ts b/src/utils/respond.ts index 0324702..7eb1796 100644 --- a/src/utils/respond.ts +++ b/src/utils/respond.ts @@ -74,10 +74,21 @@ export const withCache = (ctx: Context, age: number, immutable = false) => { /** * Check if the request is from the website base URL, and should get a React response, instead of a JSON response for API requests. * + * Supports a wildcard port to be used for local development. + * * @param ctx Request context. */ -export const isWebsite = (ctx: Context) => - env.WEBSITE_BASE && ctx.req.url.startsWith(env.WEBSITE_BASE); +export const isWebsite = (ctx: Context) => { + if (!env.WEBSITE_BASE) return false; + + const { origin } = new URL(ctx.req.url); + return ( + origin === env.WEBSITE_BASE || + (env.WEBSITE_BASE.endsWith(':*') && + origin.replace(/:\d+$/, '') === + env.WEBSITE_BASE.replace(/:\*$/, '')) + ); +}; /** * Respond to a request with data, handling if it should be returned as JSON or rendered as a React response. diff --git a/src/utils/spec/playwright.ts b/src/utils/spec/playwright.ts index 6aaa802..bb5b58b 100644 --- a/src/utils/spec/playwright.ts +++ b/src/utils/spec/playwright.ts @@ -2,9 +2,6 @@ import { test as base, expect } from '@playwright/test'; import { type TestHarness, createTestHarness } from 'wrangler'; const configPath = './wrangler.jsonc'; -// isWebsite() uses startsWith(), so the trailing colon matches every -// dynamically allocated loopback port without matching another host. -const websiteBase = 'http://127.0.0.1:'; interface TestFixtures { reset: undefined; @@ -19,7 +16,10 @@ export const createServer = (vars?: Record) => workers: [ { configPath, - vars: { WEBSITE_BASE: websiteBase, ...vars }, + vars: { + WEBSITE_BASE: 'http://127.0.0.1:*', + ...vars, + }, }, ], }); @@ -55,7 +55,7 @@ export const test = base.extend({ } const { url } = await server.listen(); - await use(url.href); + await use(url.origin); }, reset: [ async ({ server }, use, testInfo) => {