From 3ef77ae2bd0eaa94634ccdb5fd9bdaaa94bdb99f Mon Sep 17 00:00:00 2001 From: ZJ van de Weg Date: Sat, 8 Aug 2026 15:45:14 -0700 Subject: [PATCH 1/3] Add lastmod/images to sitemap, fix docs missing entirely Enriches the docs/handbook/changelog/blog/ebooks/whitepapers sitemap entries with git-derived lastmod and frontmatter images via a new Nitro sitemap source (content-urls.get.ts), since @nuxtjs/sitemap's content onUrl/filter hooks are re-spliced as raw source text with no closure over this module's imports. Also discovered docs/** had no `sitemap` schema field at all, so every /docs/** page was silently absent from sitemap.xml - fixed as part of the same route. --- nuxt/content.config.ts | 21 ++-- nuxt/lib/git-lastmod.mjs | 78 ++++++++++++ nuxt/nuxt.config.ts | 4 + .../api/__sitemap__/content-urls.get.ts | 115 ++++++++++++++++++ .../api/__sitemap__/dynamic-urls.get.ts | 5 +- 5 files changed, 208 insertions(+), 15 deletions(-) create mode 100644 nuxt/lib/git-lastmod.mjs create mode 100644 nuxt/server/api/__sitemap__/content-urls.get.ts diff --git a/nuxt/content.config.ts b/nuxt/content.config.ts index b82fc55749..5bf18a5620 100644 --- a/nuxt/content.config.ts +++ b/nuxt/content.config.ts @@ -1,6 +1,5 @@ import { join } from 'node:path' import { defineContentConfig, defineCollection, z } from '@nuxt/content' -import { defineSitemapSchema } from '@nuxtjs/sitemap/content' const tierValue = z.object({ value: z.union([z.boolean(), z.null(), z.string()]), @@ -45,6 +44,12 @@ export default defineContentConfig({ meta: z.object({ description: z.string().optional(), }).optional(), + // No `sitemap` schema field here on purpose - @nuxtjs/sitemap's own + // @nuxt/content integration only accepts *plain* onUrl/filter functions + // (it re-splices their source text into a generated file with no closure + // over this module's imports/helpers), which git-based lastmod needs. + // docs/handbook/changelog/blog/ebooks/whitepapers are all enriched instead + // by server/api/__sitemap__/content-urls.get.ts, a normal Nitro route. }) }), handbook: defineCollection({ @@ -59,7 +64,6 @@ export default defineContentConfig({ // here @nuxt/content strips the key from frontmatter. order: z.number().optional(), }).optional(), - sitemap: defineSitemapSchema(), }) }), // The Application Guide pages are strongly structured rather than free prose: each page @@ -96,7 +100,6 @@ export default defineContentConfig({ date: z.coerce.date(), authors: z.array(z.string()).optional(), issues: z.array(z.string()).optional(), - sitemap: defineSitemapSchema(), }) }), // Source files stay at src/blog/ (11ty's historical location) rather than @@ -141,7 +144,6 @@ export default defineContentConfig({ })).optional(), }).optional(), }).optional(), - sitemap: defineSitemapSchema(), }) }), ebooks: defineCollection({ @@ -165,7 +167,6 @@ export default defineContentConfig({ reference: z.string().optional(), }), contentTable: z.array(z.string()), - sitemap: defineSitemapSchema(), }) }), whitepapers: defineCollection({ @@ -189,14 +190,8 @@ export default defineContentConfig({ whitepaperSubtitle: z.string().optional(), formTitle: z.string().optional(), formSubtitle: z.string().optional(), - // Content lives under /whitepapers/* but the page route is singular: - // /whitepaper/[slug].vue — rewrite the sitemap loc to match. - sitemap: defineSitemapSchema({ - name: 'whitepapers', - onUrl: (url) => { - url.loc = url.loc.replace(/^\/whitepapers\//, '/whitepaper/') - }, - }), + // Content lives under /whitepapers/* but the page route is singular + // (/whitepaper/[slug].vue) - content-urls.get.ts rewrites the sitemap loc. }), }), plans: defineCollection({ diff --git a/nuxt/lib/git-lastmod.mjs b/nuxt/lib/git-lastmod.mjs new file mode 100644 index 0000000000..91a5d20729 --- /dev/null +++ b/nuxt/lib/git-lastmod.mjs @@ -0,0 +1,78 @@ +// Resolves a file's sitemap `lastmod` from git history rather than build time, so a +// static-generated site doesn't stamp every URL with the deploy timestamp. Builds the +// whole repo-root -> commit-date map in one `git log` walk (memoized per repoRoot) +// instead of spawning `git log` per file, which would be one process per content page - +// the same N+1 GitLab hit in Gitaly's ListLastCommitsForTree RPC (used to render a +// "last commit" column per file in the web file browser), which originally ran +// `git log -1` once per tree entry. GitLab's fix - upstreamed into git.git as the +// `git last-modified` builtin (Git 2.52, née GitHub's internal "blame-tree", used for +// GitHub's own file-browser column since 2012) - walks commit history once and, at each +// commit, diffs trees to see which of the *still-unresolved* requested paths changed, +// pruning whole unchanged subtrees via tree-object-id ("treesame") comparisons instead of +// diffing file-by-file, and stops early once every path has been resolved. That is a +// more efficient shared walk (matches Gitaly/GitHub's write-up: revisiting the same +// commits per file is "twice the necessary" work); this module's `--name-only` walk +// below shares the same core insight - one walk, not one process per file, and the first +// occurrence of a path scanning newest-first is its most recent commit - but reads the +// whole history as flat text and dedupes in JS rather than pruning the walk itself, which +// is fine at this repo's history size (a few seconds, once, memoized) but wouldn't scale +// to a git-last-modified-sized monorepo. +import { execFileSync } from 'node:child_process' + +// A NUL byte can't appear in a file path, so it safely marks a commit-date line apart +// from the `--name-only` file lines that follow it. `%x00` asks git to emit the byte +// into its output; the argv string itself only ever contains the ASCII text "%x00". +const NUL = '\u0000' +const mapCache = new Map() + +/** + * Pure parser for `git log --pretty=format:%x00%ci --name-only` output - split out from + * buildLastmodMap so the newest-first/first-occurrence-wins logic can be unit tested + * against fixture strings without shelling out to git or touching a real repo. + * + * @param {string} output raw stdout from the git log invocation above + * @returns {Map} file path -> most recent commit date + */ +export function parseGitLogOutput (output) { + const map = new Map() + let currentDate = null + for (const line of output.split('\n')) { + if (line.startsWith(NUL)) { + currentDate = line.slice(1) + continue + } + if (!line || map.has(line)) continue + map.set(line, currentDate) + } + return map +} + +function buildLastmodMap (repoRoot) { + let output + try { + // Newest-first (git log's default order): the first time a path is seen while + // walking top-to-bottom is its most recent commit. + output = execFileSync( + 'git', + ['log', '--pretty=format:%x00%ci', '--name-only'], + { cwd: repoRoot, encoding: 'utf8', maxBuffer: 1024 * 1024 * 256 } + ) + } catch (err) { + console.warn(`[sitemap] git log failed in ${repoRoot}, lastmod will be omitted: ${err.message}`) + return new Map() + } + + return parseGitLogOutput(output) +} + +/** + * @param {string} repoRoot absolute path to the git repository root + * @param {string} relativePath path to the file, relative to repoRoot (e.g. "src/blog/2024/01/post.md") + * @returns {string|undefined} the ISO-ish commit date git log reports, or undefined if unknown + */ +export function getGitLastmod (repoRoot, relativePath) { + if (!mapCache.has(repoRoot)) { + mapCache.set(repoRoot, buildLastmodMap(repoRoot)) + } + return mapCache.get(repoRoot).get(relativePath) +} diff --git a/nuxt/nuxt.config.ts b/nuxt/nuxt.config.ts index 2f4384df38..3227b06d61 100644 --- a/nuxt/nuxt.config.ts +++ b/nuxt/nuxt.config.ts @@ -212,6 +212,10 @@ export default defineNuxtConfig({ // Nuxt-native dynamic routes (integrations) that // @nuxtjs/seo's static-route auto-discovery can't see '/api/__sitemap__/dynamic-urls', + // docs/handbook/changelog/blog/ebooks/whitepapers with lastmod/images - + // see content-urls.get.ts for why this isn't done via a `sitemap` schema + // field on the collections instead. + '/api/__sitemap__/content-urls', ], urls: blogAuthorRoutes.map(loc => ({ loc, priority: 0.6 })), exclude: ['/_studio/**', '/api/**'], diff --git a/nuxt/server/api/__sitemap__/content-urls.get.ts b/nuxt/server/api/__sitemap__/content-urls.get.ts new file mode 100644 index 0000000000..5d7a4ec0a8 --- /dev/null +++ b/nuxt/server/api/__sitemap__/content-urls.get.ts @@ -0,0 +1,115 @@ +import { existsSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { queryCollection } from '@nuxt/content/server' +import { getGitLastmod } from '../../../lib/git-lastmod.mjs' + +// docs/handbook/changelog/blog/ebooks/whitepapers deliberately carry no `sitemap` schema +// field in content.config.ts - @nuxtjs/sitemap's @nuxt/content integration re-splices +// each collection's onUrl/filter as raw source text into a generated file with no closure +// over this module's imports or helpers, which git-based lastmod needs. This route does +// the same job (loc/lastmod/images) as a normal Nitro handler instead. + +const SITE_URL = 'https://flowfuse.com' +const toAbsoluteUrl = (path: string) => (path.startsWith('http') ? path : `${SITE_URL}${path}`) + +// Runs at prerender time (during `nuxt generate`), inside the git checkout, so walking up +// from cwd to find `.git` is robust whether the build is invoked from the repo root or the +// `nuxt/` workspace - unlike at request time in the deployed function, where no .git exists. +function findRepoRoot (start: string): string { + let dir = start + for (let i = 0; i < 6; i++) { + if (existsSync(join(dir, '.git'))) return dir + const parent = dirname(dir) + if (parent === dir) break + dir = parent + } + return start +} + +const REPO_ROOT = findRepoRoot(process.cwd()) + +type ContentEntry = Record & { path?: string, stem: string } +type SitemapUrl = { loc: string, lastmod?: Date, images?: { loc: string }[] } + +interface ContentSource { + collection: string + // Repo-relative directory the collection's files live in, for git-log lastmod. + // Omit when a collection derives lastmod another way (e.g. docs' `updated` field). + fileRoot?: string + filter?: (entry: ContentEntry) => boolean + lastmod?: (entry: ContentEntry) => string | undefined + images?: (entry: ContentEntry) => string[] + rewriteLoc?: (loc: string) => string +} + +const stringField = (entry: ContentEntry, key: string) => { + const value = entry[key] + return typeof value === 'string' && value ? value : undefined +} + +const CONTENT_SOURCES: ContentSource[] = [ + { + collection: 'docs', + // Already git-derived once, at docs-sync time, against the flowfuse/flowfuse repo + // this content came from - not this repo's history. + lastmod: entry => stringField(entry, 'updated'), + // Redirect stub pages (section index pages that just 301 elsewhere) aren't a + // real destination. + filter: entry => entry.layout !== 'redirect', + }, + { collection: 'handbook', fileRoot: 'nuxt/content' }, + { collection: 'changelog', fileRoot: 'src' }, + { + collection: 'blog', + fileRoot: 'src', + images: entry => [stringField(entry, 'image')].filter((path): path is string => Boolean(path)), + }, + { + collection: 'ebooks', + fileRoot: 'nuxt/content', + images: entry => ['image', 'coverImage', 'thumbnail', 'secondaryImage', 'tertiaryImage'] + .map(key => stringField(entry, key)) + .filter((path): path is string => Boolean(path)), + }, + { + collection: 'whitepapers', + fileRoot: 'nuxt/content', + images: entry => ['image', 'thumbnail'] + .map(key => stringField(entry, key)) + .filter((path): path is string => Boolean(path)), + rewriteLoc: loc => loc.replace(/^\/whitepapers\//, '/whitepaper/'), + }, +] + +export default defineSitemapEventHandler(async (event) => { + const urls: SitemapUrl[] = [] + + for (const source of CONTENT_SOURCES) { + try { + const entries = await queryCollection(event, source.collection as never).all() as ContentEntry[] + for (const entry of entries) { + // `.navigation` entries are @nuxt/content's per-directory nav metadata, + // not real pages - @nuxtjs/sitemap's own content integration excludes + // them the same way. + if (!entry.path || entry.path.endsWith('.navigation')) continue + if (source.filter && !source.filter(entry)) continue + + const url: SitemapUrl = { loc: source.rewriteLoc ? source.rewriteLoc(entry.path) : entry.path } + + const lastmod = source.lastmod + ? source.lastmod(entry) + : (source.fileRoot ? getGitLastmod(REPO_ROOT, `${source.fileRoot}/${entry.stem}.md`) : undefined) + if (lastmod) url.lastmod = new Date(lastmod) + + const images = source.images?.(entry) ?? [] + if (images.length) url.images = images.map(loc => ({ loc: toAbsoluteUrl(loc) })) + + urls.push(url) + } + } catch (err) { + console.error(`[sitemap] failed to query "${source.collection}" collection for content-urls:`, err) + } + } + + return urls +}) diff --git a/nuxt/server/api/__sitemap__/dynamic-urls.get.ts b/nuxt/server/api/__sitemap__/dynamic-urls.get.ts index 40120b4b22..7b5943d740 100644 --- a/nuxt/server/api/__sitemap__/dynamic-urls.get.ts +++ b/nuxt/server/api/__sitemap__/dynamic-urls.get.ts @@ -1,8 +1,9 @@ import { selectTopIntegrationNodes } from '../../utils/integrations-enrich' // Integrations detail pages aren't @nuxt/content — they're built from the npm/GitHub -// catalogue at request/build time — so they can't carry a `sitemap` schema field like -// handbook/ebooks/whitepapers do. Enumerate them explicitly instead. +// catalogue at request/build time, so there's no collection entry to enumerate them +// from at all (see content-urls.get.ts for the @nuxt/content collections). Enumerate +// them explicitly instead. // // Deliberately uses selectTopIntegrationNodes() (one cached catalogue fetch), not // buildEnrichedIntegrations() (which also fetches npm/GitHub README/examples per node) — From 21479291c3f5bac11f93c0e086f4cdce235aaa44 Mon Sep 17 00:00:00 2001 From: ZJ van de Weg Date: Tue, 11 Aug 2026 10:09:47 -0700 Subject: [PATCH 2/3] Use site.url from nuxt-site-config instead of a duplicated SITE_URL constant Addresses Yndira's review comment on PR #5546 - site.url was already defined in nuxt.config.ts, so this route's hardcoded copy was a second source of truth for the same value. --- nuxt/server/api/__sitemap__/content-urls.get.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nuxt/server/api/__sitemap__/content-urls.get.ts b/nuxt/server/api/__sitemap__/content-urls.get.ts index 5d7a4ec0a8..f8f8392779 100644 --- a/nuxt/server/api/__sitemap__/content-urls.get.ts +++ b/nuxt/server/api/__sitemap__/content-urls.get.ts @@ -9,8 +9,8 @@ import { getGitLastmod } from '../../../lib/git-lastmod.mjs' // over this module's imports or helpers, which git-based lastmod needs. This route does // the same job (loc/lastmod/images) as a normal Nitro handler instead. -const SITE_URL = 'https://flowfuse.com' -const toAbsoluteUrl = (path: string) => (path.startsWith('http') ? path : `${SITE_URL}${path}`) +const toAbsoluteUrl = (event: Parameters[0], path: string) => + (path.startsWith('http') ? path : `${getSiteConfig(event).url}${path}`) // Runs at prerender time (during `nuxt generate`), inside the git checkout, so walking up // from cwd to find `.git` is robust whether the build is invoked from the repo root or the @@ -102,7 +102,7 @@ export default defineSitemapEventHandler(async (event) => { if (lastmod) url.lastmod = new Date(lastmod) const images = source.images?.(entry) ?? [] - if (images.length) url.images = images.map(loc => ({ loc: toAbsoluteUrl(loc) })) + if (images.length) url.images = images.map(loc => ({ loc: toAbsoluteUrl(event, loc) })) urls.push(url) } From 5b96916adb37a98864cbed15828462da6e69a73c Mon Sep 17 00:00:00 2001 From: ZJ van de Weg Date: Tue, 11 Aug 2026 10:24:45 -0700 Subject: [PATCH 3/3] Fix sitemap lastmod resolving to undefined on the deployed function Addresses https://github.com/FlowFuse/website/pull/5546#issuecomment-5252042563 /sitemap.xml wasn't in the explicit nitro.prerender.routes list, and @nuxtjs/sitemap only self-registers a route for static baking when isNuxtGenerate() is true - which checks nitro.static/preset "static", not the hybrid netlify preset this site uses. So /sitemap.xml was being served live by the deployed function instead, where /var/task has no git binary, silently dropping every git-derived lastmod (handbook, changelog, blog, ebooks, whitepapers). Explicitly prerendering it bakes it at build time instead, inside the git checkout, same as the other generated feed routes already in that list (/blog/index.xml etc). Also make blog prefer its own `lastUpdated` frontmatter field over the git-derived date, matching what the blog page itself already shows as "Updated" - so an editorial update date isn't silently overridden by whatever last touched the file for unrelated reasons. --- nuxt/nuxt.config.ts | 9 +++++++++ nuxt/server/api/__sitemap__/content-urls.get.ts | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/nuxt/nuxt.config.ts b/nuxt/nuxt.config.ts index 3227b06d61..8ee026c7db 100644 --- a/nuxt/nuxt.config.ts +++ b/nuxt/nuxt.config.ts @@ -320,6 +320,15 @@ export default defineNuxtConfig({ '/integrations', '/pricing', '/product', + // Without this, @nuxtjs/sitemap only bakes /sitemap.xml statically when + // isNuxtGenerate() is true, which checks for nitro.static/preset "static" - + // the netlify preset here is hybrid (prerendered pages + a fallback + // function), so it doesn't qualify and /sitemap.xml gets served live by + // that function instead. There, /var/task has no `git` binary, so every + // git-derived lastmod (handbook/changelog/blog/ebooks/whitepapers, see + // content-urls.get.ts) silently resolves to undefined. Explicitly listing + // it here bakes it at build time instead, inside the git checkout. + '/sitemap.xml', '/ebooks/beginner-guide-to-a-professional-nodered/', '/ebooks/ultimate-guide-to-building-applications-with-flowfuse-dashboard-for-node-red/', '/whitepaper/uns-decoupling-data-producers-and-consumers/', diff --git a/nuxt/server/api/__sitemap__/content-urls.get.ts b/nuxt/server/api/__sitemap__/content-urls.get.ts index f8f8392779..317bee115e 100644 --- a/nuxt/server/api/__sitemap__/content-urls.get.ts +++ b/nuxt/server/api/__sitemap__/content-urls.get.ts @@ -62,6 +62,11 @@ const CONTENT_SOURCES: ContentSource[] = [ { collection: 'blog', fileRoot: 'src', + // Prefer the author-set `lastUpdated` frontmatter field - the same one the blog + // page itself uses for its "Updated" date and JSON-LD dateModified - over the git + // history date, so an editorial update date isn't silently overridden by whatever + // last touched the file (a repo-wide lint fix, a typo pass elsewhere, etc). + lastmod: entry => stringField(entry, 'lastUpdated') ?? getGitLastmod(REPO_ROOT, `src/${entry.stem}.md`), images: entry => [stringField(entry, 'image')].filter((path): path is string => Boolean(path)), }, {