diff --git a/package-lock.json b/package-lock.json index 9ee9e65..73c225c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14,6 +14,7 @@ "@sentry/cloudflare": "^10.69.0", "@tanstack/react-virtual": "^3.14.9", "algoliasearch": "^5.56.0", + "fast-xml-builder": "^1.3.1", "hono": "^4.12.34", "is-deflate": "^1.0.0", "is-gzip": "^2.0.0", @@ -41,6 +42,7 @@ "eslint": "^10.8.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-jsdoc": "^63.3.2", + "fast-xml-parser": "^5.11.0", "husky": "^9.1.7", "lint-staged": "^17.2.0", "prettier": "^3.9.6", @@ -1865,6 +1867,19 @@ "@emnapi/runtime": "^1.7.1 || ^2.0.0-alpha.3" } }, + "node_modules/@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/nodable" + } + ], + "license": "MIT" + }, "node_modules/@opentelemetry/api": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", @@ -3609,6 +3624,19 @@ "node": ">= 14.0.0" } }, + "node_modules/anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/apg-lite": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/apg-lite/-/apg-lite-1.0.5.tgz", @@ -4544,6 +4572,46 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "node_modules/fast-xml-parser": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.0.tgz", + "integrity": "sha512-9IGxMqvqLOnqP+Egi1nqDHKv5k8aZ7r9n558enxcucmyVGEBNPAU+MOg/8jPIS7rO7sSq4gFm1/nHtiaubMruw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + }, + "bin": { + "fxparser": "src/cli/cli.js" + } + }, "node_modules/fault": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", @@ -5107,6 +5175,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT" + }, "node_modules/isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -5934,6 +6015,21 @@ "node": ">=8" } }, + "node_modules/path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=14.0.0" + } + }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -6714,6 +6810,22 @@ "node": ">=0.6.19" } }, + "node_modules/strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "dependencies": { + "anynum": "^1.0.1" + } + }, "node_modules/stylis": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", @@ -7446,6 +7558,21 @@ "repeat-string": "^1.5.2" } }, + "node_modules/xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/NaturalIntelligence" + } + ], + "license": "MIT", + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", @@ -8538,6 +8665,12 @@ "@tybys/wasm-util": "^0.10.3" } }, + "@nodable/entities": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/@nodable/entities/-/entities-3.0.0.tgz", + "integrity": "sha512-8L9xFeTYKhm49xfIypoe2W5wV1m/3Z58kT+7kR9A8OyFxcPduI4VmxaUMQyKYrRjUoLLSXv6EKKID5Tvj9cUVw==", + "dev": true + }, "@opentelemetry/api": { "version": "1.9.1", "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.9.1.tgz", @@ -9847,6 +9980,12 @@ "@algolia/requester-node-http": "5.56.0" } }, + "anynum": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/anynum/-/anynum-1.0.1.tgz", + "integrity": "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A==", + "dev": true + }, "apg-lite": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/apg-lite/-/apg-lite-1.0.5.tgz", @@ -10475,6 +10614,29 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-xml-builder": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/fast-xml-builder/-/fast-xml-builder-1.3.1.tgz", + "integrity": "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug==", + "requires": { + "path-expression-matcher": "^1.6.2", + "xml-naming": "^0.3.0" + } + }, + "fast-xml-parser": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/fast-xml-parser/-/fast-xml-parser-5.11.0.tgz", + "integrity": "sha512-9IGxMqvqLOnqP+Egi1nqDHKv5k8aZ7r9n558enxcucmyVGEBNPAU+MOg/8jPIS7rO7sSq4gFm1/nHtiaubMruw==", + "dev": true, + "requires": { + "@nodable/entities": "^3.0.0", + "fast-xml-builder": "^1.2.0", + "is-unsafe": "^2.0.0", + "path-expression-matcher": "^1.6.2", + "strnum": "^2.4.2", + "xml-naming": "^0.3.0" + } + }, "fault": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/fault/-/fault-1.0.4.tgz", @@ -10806,6 +10968,12 @@ "which-typed-array": "^1.1.16" } }, + "is-unsafe": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-unsafe/-/is-unsafe-2.0.2.tgz", + "integrity": "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ==", + "dev": true + }, "isarray": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/isarray/-/isarray-2.0.5.tgz", @@ -11294,6 +11462,11 @@ "integrity": "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==", "dev": true }, + "path-expression-matcher": { + "version": "1.6.2", + "resolved": "https://registry.npmjs.org/path-expression-matcher/-/path-expression-matcher-1.6.2.tgz", + "integrity": "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ==" + }, "path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -11794,6 +11967,15 @@ "integrity": "sha512-aqD2Q0144Z+/RqG52NeHEkZauTAUWJO8c6yTftGJKO3Tja5tUgIfmIl6kExvhtxSDP7fXB6DvzkfMpCd/F3G+Q==", "dev": true }, + "strnum": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/strnum/-/strnum-2.4.2.tgz", + "integrity": "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw==", + "dev": true, + "requires": { + "anynum": "^1.0.1" + } + }, "stylis": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/stylis/-/stylis-4.2.0.tgz", @@ -12240,6 +12422,11 @@ "repeat-string": "^1.5.2" } }, + "xml-naming": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/xml-naming/-/xml-naming-0.3.0.tgz", + "integrity": "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ==" + }, "yaml": { "version": "2.9.0", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", diff --git a/package.json b/package.json index 6542d17..2de7cf1 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "@sentry/cloudflare": "^10.69.0", "@tanstack/react-virtual": "^3.14.9", "algoliasearch": "^5.56.0", + "fast-xml-builder": "^1.3.1", "hono": "^4.12.34", "is-deflate": "^1.0.0", "is-gzip": "^2.0.0", @@ -65,6 +66,7 @@ "eslint": "^10.8.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-jsdoc": "^63.3.2", + "fast-xml-parser": "^5.11.0", "husky": "^9.1.7", "lint-staged": "^17.2.0", "prettier": "^3.9.6", diff --git a/src/routes/index.browser.spec.ts b/src/routes/index.browser.spec.ts index ef74ec5..b49a2ed 100644 --- a/src/routes/index.browser.spec.ts +++ b/src/routes/index.browser.spec.ts @@ -1,4 +1,26 @@ -import { expect, test } from '../utils/spec/playwright.ts'; +import { XMLParser } from 'fast-xml-parser'; + +import { createServer, expect, test } from '../utils/spec/playwright.ts'; + +interface Sitemap { + urlset?: { + '@_xmlns'?: string; + url?: { loc?: string } | { loc?: string }[]; + }; +} + +interface OpenSearch { + OpenSearchDescription?: { + '@_xmlns'?: string; + '@_xmlns:moz'?: string; + ShortName?: string; + Url?: { + '@_type'?: string; + '@_method'?: string; + '@_template'?: string; + }; + }; +} test.describe('/', () => { test('renders page', async ({ page }) => { @@ -65,8 +87,17 @@ test.describe('/health', () => { }); test.describe('/robots.txt', () => { - test('valid response', async ({ page }) => { - const response = await page.request.get('/robots.txt'); + const isProduction = + process.env.PLAYWRIGHT_EXTERNAL_WEB_URL?.replace(/\/+$/, '') === + 'https://cdnjs.com'; + + test('blocks indexing on non-production websites', async ({ page }) => { + test.skip( + isProduction, + 'Production website robots.txt does not block indexing', + ); + + const response = await page.goto('/robots.txt'); expect(response?.ok()).toBe(true); expect(response?.status()).toBe(200); expect(response?.headers()['cache-control']).toBe( @@ -77,6 +108,49 @@ test.describe('/robots.txt', () => { ); expect(await response?.text()).toBe('User-agent: *\nDisallow: /'); }); + + test('allows indexing on production website', async ({ page }) => { + // If we're not running against production, map the /robots.txt request + // to a test server that simulates the production environment + let cleanup = () => Promise.resolve(); + if (!isProduction) { + const server = createServer({ WEBSITE_BASE: 'https://cdnjs.com' }); + await server.listen(); + + const route = await page.route('/robots.txt', async (route) => { + const response = await server + .getWorker() + .fetch('https://cdnjs.com/robots.txt'); + route.fulfill({ + status: response.status, + headers: Object.fromEntries(response.headers.entries()), + body: await response.text(), + }); + }); + + cleanup = async () => { + await route.dispose(); + await server.close(); + }; + } + + try { + const response = await page.goto('/robots.txt'); + expect(response?.ok()).toBe(true); + expect(response?.status()).toBe(200); + expect(response?.headers()['cache-control']).toBe( + 'public, max-age=30672000, immutable', + ); // 355 days + expect(response?.headers()['content-type']).toMatch( + /^text\/plain(;|$)/, + ); + expect(await response?.text()).toBe( + 'User-agent: *\nAllow: /\nSitemap: https://cdnjs.com/sitemap.xml', + ); + } finally { + await cleanup(); + } + }); }); test.describe('/opensearch.xml', () => { @@ -90,8 +164,62 @@ test.describe('/opensearch.xml', () => { expect(response?.headers()['content-type']).toMatch( /^application\/opensearchdescription\+xml(;|$)/, ); - expect(await response?.text()).toContain( - '/libraries?search={searchTerms}', + + const body = await response.text(); + const origin = new URL(response.url()).origin; + const opensearch = new XMLParser({ + ignoreAttributes: false, + }).parse(body) as unknown as OpenSearch; + const description = opensearch.OpenSearchDescription; + + expect(description?.['@_xmlns']).toBe( + 'http://a9.com/-/spec/opensearch/1.1/', + ); + expect(description?.['@_xmlns:moz']).toBe( + 'http://www.mozilla.org/2006/browser/search/', + ); + expect(description?.ShortName).toBe('cdnjs'); + expect(description?.Url?.['@_type']).toBe('text/html'); + expect(description?.Url?.['@_method']).toBe('GET'); + expect(description?.Url?.['@_template']).toBe( + `${origin}/libraries?search={searchTerms}`, + ); + }); +}); + +test.describe('/sitemap.xml', () => { + test('valid response', async ({ page }) => { + const response = await page.request.get('/sitemap.xml'); + expect(response?.ok()).toBe(true); + expect(response?.status()).toBe(200); + expect(response?.headers()['cache-control']).toBe( + 'public, max-age=21600', + ); // 6 hours + expect(response?.headers()['content-type']).toMatch( + /^application\/xml(;|$)/, + ); + + const body = await response.text(); + const origin = new URL(response.url()).origin; + const sitemap = new XMLParser({ + ignoreAttributes: false, + }).parse(body) as unknown as Sitemap; + const urls = sitemap.urlset?.url; + + expect(sitemap.urlset?.['@_xmlns']).toBe( + 'http://www.sitemaps.org/schemas/sitemap/0.9', + ); + expect(Array.isArray(urls)).toBe(true); + if (!Array.isArray(urls)) throw new Error('Missing sitemap URLs'); + + const locations = urls.map((url) => url.loc); + expect(locations).toContain(`${origin}/`); + expect(locations).toContain(`${origin}/about`); + expect(locations).toContain(`${origin}/api`); + expect(locations).toContain(`${origin}/libraries`); + expect(locations).toContain(`${origin}/libraries/backbone.js`); + expect(locations).not.toContain( + `${origin}/libraries/backbone.js/1.1.0`, ); }); }); diff --git a/src/routes/index.spec.ts b/src/routes/index.spec.ts index c2e7764..08a63af 100644 --- a/src/routes/index.spec.ts +++ b/src/routes/index.spec.ts @@ -96,20 +96,38 @@ describe('/opensearch.xml', () => { // Test the endpoint testCors(path, response); - it('returns the correct Cache headers', () => { - expect(response.headers.get('Cache-Control')).to.eq( - 'public, max-age=30672000, immutable', - ); // 355 days + it('returns the correct status code', () => { + expect(response.status).to.eq(404); + }); + it('does not return an OpenSearch description', async () => { + expect(response.headers.get('Content-Type')).to.match( + /application\/json/, + ); + expect(await response.json()).to.have.property( + 'message', + 'Endpoint not found', + ); }); + }); +}); + +describe('/sitemap.xml', () => { + const path = '/sitemap.xml'; + + describe('No query params', () => { + const response = beforeRequest(path); + + testCors(path, response); it('returns the correct status code', () => { - expect(response.status).to.eq(200); + expect(response.status).to.eq(404); }); - it('includes the libraries endpoint for searching', async () => { + it('does not return a sitemap', async () => { expect(response.headers.get('Content-Type')).to.match( - /application\/opensearchdescription\+xml/, + /application\/json/, ); - expect(await response.text()).to.include( - `/libraries?search={searchTerms}`, + expect(await response.json()).to.have.property( + 'message', + 'Endpoint not found', ); }); }); diff --git a/src/routes/index.ts b/src/routes/index.ts index 9d3488d..7d32231 100644 --- a/src/routes/index.ts +++ b/src/routes/index.ts @@ -1,11 +1,14 @@ import type { OpenAPIRegistry } from '@asteasolutions/zod-to-openapi'; +import { env } from 'cloudflare:workers'; +import XMLBuilder from 'fast-xml-builder'; import type { Context, Hono } from 'hono'; import bannerPng from '../assets/banner.png'; import faviconIco from '../assets/favicon.ico'; import faviconPng from '../assets/favicon.png'; import faviconSvg from '../assets/favicon.svg'; -import respond, { isWebsite, withCache } from '../utils/respond.ts'; +import { libraries } from '../utils/metadata.ts'; +import respond, { isWebsite, notFound, withCache } from '../utils/respond.ts'; import IndexPage from './index.page.tsx'; @@ -54,6 +57,13 @@ const handleGetRobotsTxt = (ctx: Context) => { // This is also immutable withCache(ctx, 355 * 24 * 60 * 60, true); + // Only the production site at cdnjs.com should ever be indexable + if (env.WEBSITE_BASE === 'https://cdnjs.com' && isWebsite(ctx)) { + return ctx.text( + 'User-agent: *\nAllow: /\nSitemap: https://cdnjs.com/sitemap.xml', + ); + } + // Disallow all robots return ctx.text('User-agent: *\nDisallow: /'); }; @@ -64,28 +74,99 @@ const handleGetRobotsTxt = (ctx: Context) => { * @param ctx Request context. */ const handleGetOpenSearchXml = (ctx: Context) => { + // Only serve OpenSearch for website requests + if (!isWebsite(ctx)) { + return notFound(ctx, 'Endpoint'); + } + // Set a 355 day (same as CDN) life on this response // This is also immutable withCache(ctx, 355 * 24 * 60 * 60, true); - // Respond const origin = new URL(ctx.req.url).origin; - return ctx.body( - ` - - cdnjs - cdnjs is the free, open-source CDN for the web's most popular libraries. JavaScript, CSS, and font resources, globally cached on Cloudflare's network. Trusted by 12.5% of all websites, serving 250 billion requests per month. - UTF-8 - ${origin}/favicon.ico - ${origin}/favicon.png - ${origin}/favicon.svg - -`, - 200, - { - 'Content-Type': 'application/opensearchdescription+xml', + const opensearch = new XMLBuilder({ + format: true, + ignoreAttributes: false, + suppressEmptyNode: true, + }).build({ + '?xml': { + '@_version': '1.0', + '@_encoding': 'UTF-8', + }, + OpenSearchDescription: { + '@_xmlns': 'http://a9.com/-/spec/opensearch/1.1/', + '@_xmlns:moz': 'http://www.mozilla.org/2006/browser/search/', + ShortName: 'cdnjs', + Description: + "cdnjs is the free, open-source CDN for the web's most popular libraries. JavaScript, CSS, and font resources, globally cached on Cloudflare's network. Trusted by 12.5% of all websites, serving 250 billion requests per month.", + InputEncoding: 'UTF-8', + Image: [ + { + '@_type': 'image/x-icon', + '#text': `${origin}/favicon.ico`, + }, + { + '@_type': 'image/png', + '#text': `${origin}/favicon.png`, + }, + { + '@_type': 'image/svg+xml', + '#text': `${origin}/favicon.svg`, + }, + ], + Url: { + '@_type': 'text/html', + '@_method': 'GET', + '@_template': `${origin}/libraries?search={searchTerms}`, + }, }, + }); + + return ctx.body(opensearch, 200, { + 'Content-Type': 'application/opensearchdescription+xml', + }); +}; + +/** + * Handle GET /sitemap.xml requests. + * + * @param ctx Request context. + */ +const handleGetSitemapXml = async (ctx: Context) => { + // Only serve a sitemap for website requests + if (!isWebsite(ctx)) { + return notFound(ctx, 'Endpoint'); + } + + // Set a 6 hour life on this response + withCache(ctx, 6 * 60 * 60); + + const origin = new URL(ctx.req.url).origin; + const libraryPaths = (await libraries()).map( + (name) => `/libraries/${encodeURIComponent(name)}`, ); + const sitemap = new XMLBuilder({ + format: true, + ignoreAttributes: false, + suppressEmptyNode: true, + }).build({ + '?xml': { + '@_version': '1.0', + '@_encoding': 'UTF-8', + }, + urlset: { + '@_xmlns': 'http://www.sitemaps.org/schemas/sitemap/0.9', + url: ['/', '/about', '/api', '/libraries', ...libraryPaths].map( + (path) => ({ + loc: `${origin}${path}`, + }), + ), + }, + }); + + return ctx.body(sitemap, 200, { + 'Content-Type': 'application/xml', + }); }; /** @@ -172,6 +253,9 @@ export default (app: Hono, _registry: OpenAPIRegistry) => { // Provide OpenSearch support for the website app.get('/opensearch.xml', handleGetOpenSearchXml); + // Provide a sitemap for the website + app.get('/sitemap.xml', handleGetSitemapXml); + // Serve the favicon assets app.get('/favicon.ico', handleGetFaviconIco); app.get('/favicon.png', handleGetFaviconPng); diff --git a/src/utils/respond.ts b/src/utils/respond.ts index 0324702..7eb1796 100644 --- a/src/utils/respond.ts +++ b/src/utils/respond.ts @@ -74,10 +74,21 @@ export const withCache = (ctx: Context, age: number, immutable = false) => { /** * Check if the request is from the website base URL, and should get a React response, instead of a JSON response for API requests. * + * Supports a wildcard port to be used for local development. + * * @param ctx Request context. */ -export const isWebsite = (ctx: Context) => - env.WEBSITE_BASE && ctx.req.url.startsWith(env.WEBSITE_BASE); +export const isWebsite = (ctx: Context) => { + if (!env.WEBSITE_BASE) return false; + + const { origin } = new URL(ctx.req.url); + return ( + origin === env.WEBSITE_BASE || + (env.WEBSITE_BASE.endsWith(':*') && + origin.replace(/:\d+$/, '') === + env.WEBSITE_BASE.replace(/:\*$/, '')) + ); +}; /** * Respond to a request with data, handling if it should be returned as JSON or rendered as a React response. diff --git a/src/utils/spec/playwright.ts b/src/utils/spec/playwright.ts index 30c1051..bb5b58b 100644 --- a/src/utils/spec/playwright.ts +++ b/src/utils/spec/playwright.ts @@ -2,9 +2,6 @@ import { test as base, expect } from '@playwright/test'; import { type TestHarness, createTestHarness } from 'wrangler'; const configPath = './wrangler.jsonc'; -// isWebsite() uses startsWith(), so the trailing colon matches every -// dynamically allocated loopback port without matching another host. -const websiteBase = 'http://127.0.0.1:'; interface TestFixtures { reset: undefined; @@ -14,6 +11,19 @@ interface WorkerFixtures { server?: TestHarness; } +export const createServer = (vars?: Record) => + createTestHarness({ + workers: [ + { + configPath, + vars: { + WEBSITE_BASE: 'http://127.0.0.1:*', + ...vars, + }, + }, + ], + }); + export const test = base.extend({ server: [ async ({ browserName }, use) => { @@ -28,14 +38,7 @@ export const test = base.extend({ return; } - const server = createTestHarness({ - workers: [ - { - configPath, - vars: { WEBSITE_BASE: websiteBase }, - }, - ], - }); + const server = createServer(); try { await use(server); @@ -52,7 +55,7 @@ export const test = base.extend({ } const { url } = await server.listen(); - await use(url.href); + await use(url.origin); }, reset: [ async ({ server }, use, testInfo) => {