From 9a5c74309cbd2882d0b9c7f65557b5b54179f6c2 Mon Sep 17 00:00:00 2001
From: Mario Marquez <60635115+Mariomarquezt@users.noreply.github.com>
Date: Mon, 7 Sep 2026 06:16:25 -0700
Subject: [PATCH 1/2] refactor(publisher): move the site CSS bundle server out
of the dispatcher
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
`server/router.ts` owned `serveSiteCss` along with its `(bundle, hash)`
memo, in-flight de-duplication, and the published-snapshot rebuild walk —
about 130 lines of publishing logic in a module whose one reason is to
dispatch requests. That also left the dispatcher three lines under the
700-line ceiling `module-size-budgets.test.ts` enforces, so any new route
had to displace something first.
The block moves verbatim to `server/publish/siteCssServer.ts`, next to the
`siteCssBundle.ts` that builds what it serves; `tryServeSiteCssNamespace`
now just forwards the path. No behaviour change — same disk-first order,
same memo semantics, same responses.
---
docs/features/publisher.md | 11 ++-
docs/server.md | 3 +-
server/publish/siteCssServer.ts | 153 ++++++++++++++++++++++++++++++++
server/router.ts | 144 +-----------------------------
4 files changed, 164 insertions(+), 147 deletions(-)
create mode 100644 server/publish/siteCssServer.ts
diff --git a/docs/features/publisher.md b/docs/features/publisher.md
index f7efd7b6a..ce0cd16dc 100644
--- a/docs/features/publisher.md
+++ b/docs/features/publisher.md
@@ -52,6 +52,7 @@ server/publish/
├── publicRenderer.ts — renderPublishedSnapshot, renderPublishedDataRowTemplate
├── publishedHtmlPipeline.ts — post-process (sanitize + plugin filters + injections)
├── siteCssBundle.ts — server-side hashing + file emission
+├── siteCssServer.ts — serves `/_instatic/css/*` (disk-first, memoised DB-rebuild fallback)
├── frontendInjections.ts — splice plugin ')
+ const res = await serveRootFile('/key.txt')
+ expect(res!.headers.get('content-type')).toBe('text/plain; charset=utf-8')
+ expect(res!.headers.get('x-content-type-options')).toBe('nosniff')
+ expect(await res!.text()).toBe('')
+ })
+
+ it('returns null for a path nobody claimed', async () => {
+ claim('acme.seo', '/key.txt', 'key')
+ expect(await serveRootFile('/other.txt')).toBeNull()
+ })
+
+ it('matches the claimed path exactly — case-sensitive, no query, no trailing slash', async () => {
+ claim('acme.seo', '/AbC.txt', 'AbC')
+ expect(await serveRootFile('/abc.txt')).toBeNull()
+ expect(await serveRootFile('/AbC.txt/')).toBeNull()
+ expect(await serveRootFile('/AbC.txt')).not.toBeNull()
+ })
+
+ it('refuses claims outside the single-segment .txt allowlist', async () => {
+ for (const path of [
+ '/../secret.txt',
+ '/a/../b.txt',
+ '/nested/key.txt',
+ '/.hidden.txt',
+ '/-leading.txt',
+ '/index.html',
+ '/favicon.svg',
+ '/sitemap.xml',
+ '/key.txt.html',
+ '/',
+ 'key.txt',
+ ]) {
+ claim('acme.bad', path, 'payload')
+ expect(await serveRootFile(path)).toBeNull()
+ hookBus.reset()
+ }
+ })
+
+ it('never resolves a percent-encoded request to a claim', async () => {
+ claim('acme.seo', '/key.txt', 'key')
+ // Pathnames arrive un-decoded and a claimable path cannot contain `%`,
+ // so an encoded traversal fails the allowlist instead of matching.
+ expect(await serveRootFile('/%2e%2e%2fkey.txt')).toBeNull()
+ expect(await serveRootFile('/%6bey.txt')).toBeNull()
+ })
+
+ it('refuses a claim on the host-managed /robots.txt and says so', async () => {
+ const errorLog = spyOn(console, 'error').mockImplementation(() => {})
+ try {
+ onRootFiles('acme.bad', (doc) => {
+ doc.files.push({ path: '/robots.txt', content: 'User-agent: *\nDisallow: /' })
+ doc.files.push({ path: '/ok.txt', content: 'ok' })
+ return doc
+ })
+ expect(await serveRootFile('/robots.txt')).toBeNull()
+ // Resolving any claimable path surfaces the rejected claim; the
+ // plugin's own file is unaffected.
+ const own = await serveRootFile('/ok.txt')
+ expect(await own!.text()).toBe('ok')
+ expect(errorLog).toHaveBeenCalledWith(
+ '[siteRoot] root file "/robots.txt" is reserved by the host; ignoring the claim. ' +
+ 'Registered by one of: acme.bad',
+ )
+ } finally {
+ errorLog.mockRestore()
+ }
+ // The host document is unaffected by the attempted claim.
+ expect(await robotsBody()).toBe('User-agent: *\nAllow: /\n')
+ })
+
+ it('refuses a path two plugins claim, naming the candidates', async () => {
+ const errorLog = spyOn(console, 'error').mockImplementation(() => {})
+ try {
+ claim('acme.seo', '/key.txt', 'acme-key')
+ claim('zeta.seo', '/key.txt', 'zeta-key')
+ expect(await serveRootFile('/key.txt')).toBeNull()
+ expect(errorLog).toHaveBeenCalledWith(
+ '[siteRoot] root file "/key.txt" was claimed more than once; refusing to serve it. ' +
+ 'Registered by one of: acme.seo, zeta.seo',
+ )
+ } finally {
+ errorLog.mockRestore()
+ }
+ })
+
+ it('keeps the uncontested claims of a plugin that also contests one', async () => {
+ const errorLog = spyOn(console, 'error').mockImplementation(() => {})
+ try {
+ claim('acme.seo', '/shared.txt', 'acme')
+ onRootFiles('zeta.seo', (doc) => {
+ doc.files.push({ path: '/shared.txt', content: 'zeta' })
+ doc.files.push({ path: '/zeta-only.txt', content: 'zeta-only' })
+ return doc
+ })
+ expect(await serveRootFile('/shared.txt')).toBeNull()
+ const own = await serveRootFile('/zeta-only.txt')
+ expect(await own!.text()).toBe('zeta-only')
+ } finally {
+ errorLog.mockRestore()
+ }
+ })
+
+ it('drops content over the 4 KiB cap and keeps the valid claims', async () => {
+ onRootFiles('acme.bad', (doc) => {
+ doc.files.push({ path: '/huge.txt', content: 'x'.repeat(4097) })
+ doc.files.push({ path: '/small.txt', content: 'x'.repeat(4096) })
+ return doc
+ })
+ expect(await serveRootFile('/huge.txt')).toBeNull()
+ expect(await serveRootFile('/small.txt')).not.toBeNull()
+ })
+
+ it('drops content carrying control characters but allows tabs and newlines', async () => {
+ onRootFiles('acme.bad', (doc) => {
+ doc.files.push({ path: '/nul.txt', content: 'key\u0000padding' })
+ doc.files.push({ path: '/esc.txt', content: 'key\u001b[2J' })
+ doc.files.push({ path: '/lines.txt', content: 'line1\nline2\tcol' })
+ return doc
+ })
+ expect(await serveRootFile('/nul.txt')).toBeNull()
+ expect(await serveRootFile('/esc.txt')).toBeNull()
+ const ok = await serveRootFile('/lines.txt')
+ expect(await ok!.text()).toBe('line1\nline2\tcol')
+ })
+
+ it('discards claims past the cap so the accepted set stays bounded', async () => {
+ onRootFiles('acme.bad', (doc) => {
+ for (let i = 0; i < MAX_ROOT_FILE_CLAIMS + 5; i++) {
+ doc.files.push({ path: `/k${i}.txt`, content: `${i}` })
+ }
+ return doc
+ })
+ expect(await serveRootFile('/k0.txt')).not.toBeNull()
+ expect(await serveRootFile(`/k${MAX_ROOT_FILE_CLAIMS - 1}.txt`)).not.toBeNull()
+ expect(await serveRootFile(`/k${MAX_ROOT_FILE_CLAIMS}.txt`)).toBeNull()
+ })
+})
+
+// ---------------------------------------------------------------------------
+// Dispatcher ordering
+// ---------------------------------------------------------------------------
+
+describe('site-root files in the dispatcher', () => {
+ it('serves /robots.txt through the router', async () => {
+ const res = await handleServerRequest(new Request('http://localhost/robots.txt'), { db: fakeDb() })
+ expect(res.status).toBe(200)
+ expect(res.headers.get('content-type')).toBe('text/plain; charset=utf-8')
+ expect(await res.text()).toBe('User-agent: *\nAllow: /\n')
+ })
+
+ it('serves a claimed root file through the router', async () => {
+ claim('acme.seo', '/a1b2c3.txt', 'a1b2c3')
+ const res = await handleServerRequest(new Request('http://localhost/a1b2c3.txt'), { db: fakeDb() })
+ expect(res.status).toBe(200)
+ expect(await res.text()).toBe('a1b2c3')
+ })
+
+ it('lets an unclaimed .txt path keep falling through', async () => {
+ claim('acme.seo', '/a1b2c3.txt', 'a1b2c3')
+ const res = await handleServerRequest(new Request('http://localhost/nobody.txt'), { db: fakeDb() })
+ expect(res.status).not.toBe(200)
+ })
+
+ it('does not answer non-GET requests for the site-root paths', async () => {
+ claim('acme.seo', '/a1b2c3.txt', 'a1b2c3')
+ for (const method of ['POST', 'DELETE']) {
+ const robots = await handleServerRequest(
+ new Request('http://localhost/robots.txt', { method }),
+ { db: fakeDb() },
+ )
+ expect(robots.status).toBe(404)
+ const file = await handleServerRequest(
+ new Request('http://localhost/a1b2c3.txt', { method }),
+ { db: fakeDb() },
+ )
+ expect(file.status).toBe(404)
+ }
+ })
+
+ it('answers HEAD the same way it answers GET', async () => {
+ claim('acme.seo', '/a1b2c3.txt', 'a1b2c3')
+ const res = await handleServerRequest(
+ new Request('http://localhost/a1b2c3.txt', { method: 'HEAD' }),
+ { db: fakeDb() },
+ )
+ expect(res.status).toBe(200)
+ expect(res.headers.get('content-type')).toBe('text/plain; charset=utf-8')
+ })
+
+ it('cannot claim a path inside a host-owned namespace', async () => {
+ // `/admin/...` and `/uploads/...` are claimed by earlier routes, and the
+ // allowlist pattern rejects any path with a second segment anyway.
+ claim('acme.bad', '/uploads/evil.txt', 'payload')
+ expect(await serveRootFile('/uploads/evil.txt')).toBeNull()
+ const res = await handleServerRequest(
+ new Request('http://localhost/admin/api/cms/setup/status'),
+ { db: fakeDb() },
+ )
+ expect(res.headers.get('content-type')).toContain('application/json')
+ })
+})
diff --git a/src/core/plugin-sdk/index.ts b/src/core/plugin-sdk/index.ts
index d7ad8e382..f9960d14c 100644
--- a/src/core/plugin-sdk/index.ts
+++ b/src/core/plugin-sdk/index.ts
@@ -1,6 +1,7 @@
export * from './types'
export * from './storageSchemas'
export * from './contentSchemas'
+export * from './siteRootSchemas'
export * from './capabilities'
export * from './guards'
export * from './modules'
diff --git a/src/core/plugin-sdk/siteRootSchemas.ts b/src/core/plugin-sdk/siteRootSchemas.ts
new file mode 100644
index 000000000..1933dccdd
--- /dev/null
+++ b/src/core/plugin-sdk/siteRootSchemas.ts
@@ -0,0 +1,128 @@
+/**
+ * TypeBox schemas for the site-root text surface — the host-managed
+ * `/robots.txt` document and the root-level text files plugins can claim.
+ *
+ * These schemas are the source of truth for both `site.*` filter payloads.
+ * All types are derived from them via `Static<>`.
+ *
+ * Used across:
+ * - `src/core/plugin-sdk/types/hooks.ts` — the `CmsServerFilters` entries
+ * - `server/siteRoot.ts` — host validation + rendering
+ *
+ * Why the payloads are structured rather than raw text: a filter that
+ * returned a finished `robots.txt` string would let one plugin inject
+ * arbitrary lines (comments, `Sitemap:` entries pointing anywhere, stray
+ * CR/LF) that the host has no way to audit. Here the host owns the
+ * serialization and every field carries a pattern, so a plugin can only
+ * contribute values that render to exactly one directive line.
+ */
+
+import { Type, type Static } from '@core/utils/typeboxHelpers'
+
+// ---------------------------------------------------------------------------
+// robots.txt
+// ---------------------------------------------------------------------------
+
+/**
+ * A path prefix for an `Allow:` / `Disallow:` directive. Must start at the
+ * site root and carry no whitespace (which would split the directive line)
+ * and no `#` (which would open a robots.txt comment).
+ */
+export const RobotsPathSchema = Type.String({ pattern: '^/[^\\s#]{0,511}$' })
+
+/**
+ * A `Sitemap:` URL. Absolute by protocol requirement — a sitemap reference
+ * is resolved against the origin, not the robots.txt file.
+ */
+export const SitemapUrlSchema = Type.String({
+ maxLength: 2048,
+ pattern: '^https?://[^\\s#]+$',
+})
+
+/**
+ * Caps on how much one document can carry. The host validates list entries
+ * one by one — a group with a bad `Disallow` is dropped without costing the
+ * plugin its `Sitemap` lines — so these bounds are also what it truncates
+ * the accepted lists to; a hostile handler cannot grow the response without
+ * limit.
+ */
+export const MAX_ROBOTS_GROUPS = 20
+export const MAX_ROBOTS_SITEMAPS = 50
+const MAX_ROBOTS_PATHS_PER_GROUP = 100
+
+/**
+ * One `User-agent` group. The user-agent value is a product token (`*`,
+ * `Googlebot`), so the same no-whitespace / no-comment rule applies.
+ */
+export const RobotsGroupSchema = Type.Object(
+ {
+ userAgent: Type.String({ maxLength: 200, pattern: '^[^\\s#]+$' }),
+ allow: Type.Array(RobotsPathSchema, { maxItems: MAX_ROBOTS_PATHS_PER_GROUP }),
+ disallow: Type.Array(RobotsPathSchema, { maxItems: MAX_ROBOTS_PATHS_PER_GROUP }),
+ },
+ { additionalProperties: false },
+)
+
+export type RobotsGroup = Static
+
+/**
+ * The whole `/robots.txt` document, as the `site.robots` filter sees it.
+ * Handlers mutate and return it; the host renders the result.
+ */
+export const RobotsDocumentSchema = Type.Object(
+ {
+ groups: Type.Array(RobotsGroupSchema, { maxItems: MAX_ROBOTS_GROUPS }),
+ sitemaps: Type.Array(SitemapUrlSchema, { maxItems: MAX_ROBOTS_SITEMAPS }),
+ },
+ { additionalProperties: false },
+)
+
+export type RobotsDocument = Static
+
+// ---------------------------------------------------------------------------
+// Root-level text files
+// ---------------------------------------------------------------------------
+
+/**
+ * The paths a plugin may claim: one root segment, `.txt` only, no percent
+ * escapes, no dot segments (the leading character class rules out `.` and
+ * `-`). Deliberately narrow — the site root is shared with page slugs,
+ * the admin app, and every reserved namespace, so the claimable surface is
+ * an allowlist rather than a denylist.
+ */
+export const ROOT_FILE_PATH_PATTERN = '^/[A-Za-z0-9][A-Za-z0-9._-]{0,62}\\.txt$'
+
+/** Most claims the host accepts, and the bound it truncates the list to. */
+export const MAX_ROOT_FILE_CLAIMS = 20
+
+/**
+ * A single claimed file. `content` allows tabs and newlines but no other C0
+ * control characters, so the body can never carry NUL padding or terminal
+ * escapes; 4 KiB is well above the largest legitimate case (an IndexNow key
+ * is 8–128 characters).
+ */
+export const SiteRootFileSchema = Type.Object(
+ {
+ path: Type.String({ pattern: ROOT_FILE_PATH_PATTERN }),
+ content: Type.String({
+ maxLength: 4096,
+ pattern: '^[^\\u0000-\\u0008\\u000B\\u000C\\u000E-\\u001F]*$',
+ }),
+ },
+ { additionalProperties: false },
+)
+
+export type SiteRootFile = Static
+
+/**
+ * The claim list, as the `site.rootFiles` filter sees it. Handlers append
+ * their own entries and return it; the host resolves the claims.
+ */
+export const SiteRootFilesSchema = Type.Object(
+ {
+ files: Type.Array(SiteRootFileSchema, { maxItems: MAX_ROOT_FILE_CLAIMS }),
+ },
+ { additionalProperties: false },
+)
+
+export type SiteRootFiles = Static
diff --git a/src/core/plugin-sdk/types/hooks.ts b/src/core/plugin-sdk/types/hooks.ts
index 030a08644..3b57c64fa 100644
--- a/src/core/plugin-sdk/types/hooks.ts
+++ b/src/core/plugin-sdk/types/hooks.ts
@@ -2,6 +2,8 @@
// CMS server-side hook event surface
// ---------------------------------------------------------------------------
+import type { RobotsDocument, SiteRootFiles } from '../siteRootSchemas'
+
/**
* Actor that originated a content mutation. Carried on every
* `content.entry.*` event so listeners can filter their own writes
@@ -59,6 +61,44 @@ export interface CmsServerFilters {
* entry id (or `'new'` for create), and the actor.
*/
'content.entry.cells': Record
+ /**
+ * The host-managed `/robots.txt` document, run through every registered
+ * handler on each request for that path. The host owns the serialization,
+ * so a handler contributes directives — most often a `Sitemap:` URL —
+ * rather than text:
+ *
+ * ```ts
+ * api.cms.hooks.filter('site.robots', (doc) => {
+ * doc.sitemaps.push('https://example.com/sitemap.xml')
+ * return doc
+ * })
+ * ```
+ *
+ * The host seeds the chain with its default `User-agent: *` / `Allow: /`
+ * group, then validates every list entry: a group is accepted or dropped
+ * as a unit (`RobotsGroupSchema`), sitemap URLs one by one, and the
+ * sitemap list is de-duplicated. When the filtered document has no valid
+ * group left the host default one is re-inserted.
+ */
+ 'site.robots': RobotsDocument
+ /**
+ * Root-level text files plugins claim, run through every registered
+ * handler on each request for a `/.txt` path. Covers standards
+ * that authorize by file location — the IndexNow key file being the
+ * motivating case:
+ *
+ * ```ts
+ * api.cms.hooks.filter('site.rootFiles', (doc) => {
+ * doc.files.push({ path: `/${key}.txt`, content: key })
+ * return doc
+ * })
+ * ```
+ *
+ * Claims are validated against `SiteRootFileSchema`; `/robots.txt` is
+ * reserved for the host, and a path claimed more than once is refused
+ * rather than resolved to an arbitrary winner.
+ */
+ 'site.rootFiles': SiteRootFiles
// Plugin-defined filters fall through.
[key: string]: unknown
}
diff --git a/src/core/plugins/hookBus.ts b/src/core/plugins/hookBus.ts
index b0bc66398..c714967d1 100644
--- a/src/core/plugins/hookBus.ts
+++ b/src/core/plugins/hookBus.ts
@@ -204,6 +204,16 @@ class HookBus {
return (this.filters.get(name)?.length ?? 0) > 0
}
+ /**
+ * Plugin ids with a handler on a filter pipeline, in registration order.
+ * `applyFilter` chains handlers opaquely, so a host that has to report a
+ * bad contribution (e.g. two plugins claiming the same root file path)
+ * uses this to name the candidates in the log line.
+ */
+ pluginsFor(name: string): string[] {
+ return (this.filters.get(name) ?? []).map((entry) => entry.pluginId)
+ }
+
// Test-only introspection
__debug__(): { events: string[]; filters: string[] } {
return {