diff --git a/nuxt/content.config.ts b/nuxt/content.config.ts index b82fc55749..5bf18a5620 100644 --- a/nuxt/content.config.ts +++ b/nuxt/content.config.ts @@ -1,6 +1,5 @@ import { join } from 'node:path' import { defineContentConfig, defineCollection, z } from '@nuxt/content' -import { defineSitemapSchema } from '@nuxtjs/sitemap/content' const tierValue = z.object({ value: z.union([z.boolean(), z.null(), z.string()]), @@ -45,6 +44,12 @@ export default defineContentConfig({ meta: z.object({ description: z.string().optional(), }).optional(), + // No `sitemap` schema field here on purpose - @nuxtjs/sitemap's own + // @nuxt/content integration only accepts *plain* onUrl/filter functions + // (it re-splices their source text into a generated file with no closure + // over this module's imports/helpers), which git-based lastmod needs. + // docs/handbook/changelog/blog/ebooks/whitepapers are all enriched instead + // by server/api/__sitemap__/content-urls.get.ts, a normal Nitro route. }) }), handbook: defineCollection({ @@ -59,7 +64,6 @@ export default defineContentConfig({ // here @nuxt/content strips the key from frontmatter. order: z.number().optional(), }).optional(), - sitemap: defineSitemapSchema(), }) }), // The Application Guide pages are strongly structured rather than free prose: each page @@ -96,7 +100,6 @@ export default defineContentConfig({ date: z.coerce.date(), authors: z.array(z.string()).optional(), issues: z.array(z.string()).optional(), - sitemap: defineSitemapSchema(), }) }), // Source files stay at src/blog/ (11ty's historical location) rather than @@ -141,7 +144,6 @@ export default defineContentConfig({ })).optional(), }).optional(), }).optional(), - sitemap: defineSitemapSchema(), }) }), ebooks: defineCollection({ @@ -165,7 +167,6 @@ export default defineContentConfig({ reference: z.string().optional(), }), contentTable: z.array(z.string()), - sitemap: defineSitemapSchema(), }) }), whitepapers: defineCollection({ @@ -189,14 +190,8 @@ export default defineContentConfig({ whitepaperSubtitle: z.string().optional(), formTitle: z.string().optional(), formSubtitle: z.string().optional(), - // Content lives under /whitepapers/* but the page route is singular: - // /whitepaper/[slug].vue — rewrite the sitemap loc to match. - sitemap: defineSitemapSchema({ - name: 'whitepapers', - onUrl: (url) => { - url.loc = url.loc.replace(/^\/whitepapers\//, '/whitepaper/') - }, - }), + // Content lives under /whitepapers/* but the page route is singular + // (/whitepaper/[slug].vue) - content-urls.get.ts rewrites the sitemap loc. }), }), plans: defineCollection({ diff --git a/nuxt/lib/git-lastmod.mjs b/nuxt/lib/git-lastmod.mjs new file mode 100644 index 0000000000..91a5d20729 --- /dev/null +++ b/nuxt/lib/git-lastmod.mjs @@ -0,0 +1,78 @@ +// Resolves a file's sitemap `lastmod` from git history rather than build time, so a +// static-generated site doesn't stamp every URL with the deploy timestamp. Builds the +// whole repo-root -> commit-date map in one `git log` walk (memoized per repoRoot) +// instead of spawning `git log` per file, which would be one process per content page - +// the same N+1 GitLab hit in Gitaly's ListLastCommitsForTree RPC (used to render a +// "last commit" column per file in the web file browser), which originally ran +// `git log -1` once per tree entry. GitLab's fix - upstreamed into git.git as the +// `git last-modified` builtin (Git 2.52, née GitHub's internal "blame-tree", used for +// GitHub's own file-browser column since 2012) - walks commit history once and, at each +// commit, diffs trees to see which of the *still-unresolved* requested paths changed, +// pruning whole unchanged subtrees via tree-object-id ("treesame") comparisons instead of +// diffing file-by-file, and stops early once every path has been resolved. That is a +// more efficient shared walk (matches Gitaly/GitHub's write-up: revisiting the same +// commits per file is "twice the necessary" work); this module's `--name-only` walk +// below shares the same core insight - one walk, not one process per file, and the first +// occurrence of a path scanning newest-first is its most recent commit - but reads the +// whole history as flat text and dedupes in JS rather than pruning the walk itself, which +// is fine at this repo's history size (a few seconds, once, memoized) but wouldn't scale +// to a git-last-modified-sized monorepo. +import { execFileSync } from 'node:child_process' + +// A NUL byte can't appear in a file path, so it safely marks a commit-date line apart +// from the `--name-only` file lines that follow it. `%x00` asks git to emit the byte +// into its output; the argv string itself only ever contains the ASCII text "%x00". +const NUL = '\u0000' +const mapCache = new Map() + +/** + * Pure parser for `git log --pretty=format:%x00%ci --name-only` output - split out from + * buildLastmodMap so the newest-first/first-occurrence-wins logic can be unit tested + * against fixture strings without shelling out to git or touching a real repo. + * + * @param {string} output raw stdout from the git log invocation above + * @returns {Map} file path -> most recent commit date + */ +export function parseGitLogOutput (output) { + const map = new Map() + let currentDate = null + for (const line of output.split('\n')) { + if (line.startsWith(NUL)) { + currentDate = line.slice(1) + continue + } + if (!line || map.has(line)) continue + map.set(line, currentDate) + } + return map +} + +function buildLastmodMap (repoRoot) { + let output + try { + // Newest-first (git log's default order): the first time a path is seen while + // walking top-to-bottom is its most recent commit. + output = execFileSync( + 'git', + ['log', '--pretty=format:%x00%ci', '--name-only'], + { cwd: repoRoot, encoding: 'utf8', maxBuffer: 1024 * 1024 * 256 } + ) + } catch (err) { + console.warn(`[sitemap] git log failed in ${repoRoot}, lastmod will be omitted: ${err.message}`) + return new Map() + } + + return parseGitLogOutput(output) +} + +/** + * @param {string} repoRoot absolute path to the git repository root + * @param {string} relativePath path to the file, relative to repoRoot (e.g. "src/blog/2024/01/post.md") + * @returns {string|undefined} the ISO-ish commit date git log reports, or undefined if unknown + */ +export function getGitLastmod (repoRoot, relativePath) { + if (!mapCache.has(repoRoot)) { + mapCache.set(repoRoot, buildLastmodMap(repoRoot)) + } + return mapCache.get(repoRoot).get(relativePath) +} diff --git a/nuxt/nuxt.config.ts b/nuxt/nuxt.config.ts index 2f4384df38..8ee026c7db 100644 --- a/nuxt/nuxt.config.ts +++ b/nuxt/nuxt.config.ts @@ -212,6 +212,10 @@ export default defineNuxtConfig({ // Nuxt-native dynamic routes (integrations) that // @nuxtjs/seo's static-route auto-discovery can't see '/api/__sitemap__/dynamic-urls', + // docs/handbook/changelog/blog/ebooks/whitepapers with lastmod/images - + // see content-urls.get.ts for why this isn't done via a `sitemap` schema + // field on the collections instead. + '/api/__sitemap__/content-urls', ], urls: blogAuthorRoutes.map(loc => ({ loc, priority: 0.6 })), exclude: ['/_studio/**', '/api/**'], @@ -316,6 +320,15 @@ export default defineNuxtConfig({ '/integrations', '/pricing', '/product', + // Without this, @nuxtjs/sitemap only bakes /sitemap.xml statically when + // isNuxtGenerate() is true, which checks for nitro.static/preset "static" - + // the netlify preset here is hybrid (prerendered pages + a fallback + // function), so it doesn't qualify and /sitemap.xml gets served live by + // that function instead. There, /var/task has no `git` binary, so every + // git-derived lastmod (handbook/changelog/blog/ebooks/whitepapers, see + // content-urls.get.ts) silently resolves to undefined. Explicitly listing + // it here bakes it at build time instead, inside the git checkout. + '/sitemap.xml', '/ebooks/beginner-guide-to-a-professional-nodered/', '/ebooks/ultimate-guide-to-building-applications-with-flowfuse-dashboard-for-node-red/', '/whitepaper/uns-decoupling-data-producers-and-consumers/', diff --git a/nuxt/server/api/__sitemap__/content-urls.get.ts b/nuxt/server/api/__sitemap__/content-urls.get.ts new file mode 100644 index 0000000000..317bee115e --- /dev/null +++ b/nuxt/server/api/__sitemap__/content-urls.get.ts @@ -0,0 +1,120 @@ +import { existsSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { queryCollection } from '@nuxt/content/server' +import { getGitLastmod } from '../../../lib/git-lastmod.mjs' + +// docs/handbook/changelog/blog/ebooks/whitepapers deliberately carry no `sitemap` schema +// field in content.config.ts - @nuxtjs/sitemap's @nuxt/content integration re-splices +// each collection's onUrl/filter as raw source text into a generated file with no closure +// over this module's imports or helpers, which git-based lastmod needs. This route does +// the same job (loc/lastmod/images) as a normal Nitro handler instead. + +const toAbsoluteUrl = (event: Parameters[0], path: string) => + (path.startsWith('http') ? path : `${getSiteConfig(event).url}${path}`) + +// Runs at prerender time (during `nuxt generate`), inside the git checkout, so walking up +// from cwd to find `.git` is robust whether the build is invoked from the repo root or the +// `nuxt/` workspace - unlike at request time in the deployed function, where no .git exists. +function findRepoRoot (start: string): string { + let dir = start + for (let i = 0; i < 6; i++) { + if (existsSync(join(dir, '.git'))) return dir + const parent = dirname(dir) + if (parent === dir) break + dir = parent + } + return start +} + +const REPO_ROOT = findRepoRoot(process.cwd()) + +type ContentEntry = Record & { path?: string, stem: string } +type SitemapUrl = { loc: string, lastmod?: Date, images?: { loc: string }[] } + +interface ContentSource { + collection: string + // Repo-relative directory the collection's files live in, for git-log lastmod. + // Omit when a collection derives lastmod another way (e.g. docs' `updated` field). + fileRoot?: string + filter?: (entry: ContentEntry) => boolean + lastmod?: (entry: ContentEntry) => string | undefined + images?: (entry: ContentEntry) => string[] + rewriteLoc?: (loc: string) => string +} + +const stringField = (entry: ContentEntry, key: string) => { + const value = entry[key] + return typeof value === 'string' && value ? value : undefined +} + +const CONTENT_SOURCES: ContentSource[] = [ + { + collection: 'docs', + // Already git-derived once, at docs-sync time, against the flowfuse/flowfuse repo + // this content came from - not this repo's history. + lastmod: entry => stringField(entry, 'updated'), + // Redirect stub pages (section index pages that just 301 elsewhere) aren't a + // real destination. + filter: entry => entry.layout !== 'redirect', + }, + { collection: 'handbook', fileRoot: 'nuxt/content' }, + { collection: 'changelog', fileRoot: 'src' }, + { + collection: 'blog', + fileRoot: 'src', + // Prefer the author-set `lastUpdated` frontmatter field - the same one the blog + // page itself uses for its "Updated" date and JSON-LD dateModified - over the git + // history date, so an editorial update date isn't silently overridden by whatever + // last touched the file (a repo-wide lint fix, a typo pass elsewhere, etc). + lastmod: entry => stringField(entry, 'lastUpdated') ?? getGitLastmod(REPO_ROOT, `src/${entry.stem}.md`), + images: entry => [stringField(entry, 'image')].filter((path): path is string => Boolean(path)), + }, + { + collection: 'ebooks', + fileRoot: 'nuxt/content', + images: entry => ['image', 'coverImage', 'thumbnail', 'secondaryImage', 'tertiaryImage'] + .map(key => stringField(entry, key)) + .filter((path): path is string => Boolean(path)), + }, + { + collection: 'whitepapers', + fileRoot: 'nuxt/content', + images: entry => ['image', 'thumbnail'] + .map(key => stringField(entry, key)) + .filter((path): path is string => Boolean(path)), + rewriteLoc: loc => loc.replace(/^\/whitepapers\//, '/whitepaper/'), + }, +] + +export default defineSitemapEventHandler(async (event) => { + const urls: SitemapUrl[] = [] + + for (const source of CONTENT_SOURCES) { + try { + const entries = await queryCollection(event, source.collection as never).all() as ContentEntry[] + for (const entry of entries) { + // `.navigation` entries are @nuxt/content's per-directory nav metadata, + // not real pages - @nuxtjs/sitemap's own content integration excludes + // them the same way. + if (!entry.path || entry.path.endsWith('.navigation')) continue + if (source.filter && !source.filter(entry)) continue + + const url: SitemapUrl = { loc: source.rewriteLoc ? source.rewriteLoc(entry.path) : entry.path } + + const lastmod = source.lastmod + ? source.lastmod(entry) + : (source.fileRoot ? getGitLastmod(REPO_ROOT, `${source.fileRoot}/${entry.stem}.md`) : undefined) + if (lastmod) url.lastmod = new Date(lastmod) + + const images = source.images?.(entry) ?? [] + if (images.length) url.images = images.map(loc => ({ loc: toAbsoluteUrl(event, loc) })) + + urls.push(url) + } + } catch (err) { + console.error(`[sitemap] failed to query "${source.collection}" collection for content-urls:`, err) + } + } + + return urls +}) diff --git a/nuxt/server/api/__sitemap__/dynamic-urls.get.ts b/nuxt/server/api/__sitemap__/dynamic-urls.get.ts index 40120b4b22..7b5943d740 100644 --- a/nuxt/server/api/__sitemap__/dynamic-urls.get.ts +++ b/nuxt/server/api/__sitemap__/dynamic-urls.get.ts @@ -1,8 +1,9 @@ import { selectTopIntegrationNodes } from '../../utils/integrations-enrich' // Integrations detail pages aren't @nuxt/content — they're built from the npm/GitHub -// catalogue at request/build time — so they can't carry a `sitemap` schema field like -// handbook/ebooks/whitepapers do. Enumerate them explicitly instead. +// catalogue at request/build time, so there's no collection entry to enumerate them +// from at all (see content-urls.get.ts for the @nuxt/content collections). Enumerate +// them explicitly instead. // // Deliberately uses selectTopIntegrationNodes() (one cached catalogue fetch), not // buildEnrichedIntegrations() (which also fetches npm/GitHub README/examples per node) —