Iterate README renderer pipeline

.github/workflows/publish_after_push.yml

@@ -27,6 +27,14 @@ jobs:
         with:
           node-version: "22"
 
+      - name: Restore rendered README cache
+        uses: actions/cache@v4
+        with:
+          path: readmes_rendered.json
+          key: readmes-rendered-${{ github.run_id }}
+          restore-keys: |
+            readmes-rendered-
+
       - name: Build
         run: make build
 

eleventy.config.mjs

@@ -6,6 +6,10 @@ import * as esbuild from 'esbuild'
 import * as util from './eleventy.util.mjs'
 import * as filters from './eleventy.filters.mjs'
 import { bundleCss } from './util/bundle-css.mjs'
+import {
+  RENDERED_READMES_ENVIRONMENT_KEY,
+  createReadmeRendererEnvironmentHash,
+} from './util/readme-render-cache.mjs'
 
 const repackagerSite = 'https://repackager.sublimetext.io'
 const supportedRepackagerHosts = [
@@ -439,9 +443,20 @@ export default async function (eleventyConfig) {
 
   const workspace = JSON.parse(fs.readFileSync('workspace.json', 'utf8'))
   const stats = JSON.parse(fs.readFileSync('stats.json', 'utf8'))
-  const renderedReadmes = fs.existsSync('readmes_rendered.json')
+  let renderedReadmes = fs.existsSync('readmes_rendered.json')
     ? JSON.parse(fs.readFileSync('readmes_rendered.json', 'utf8'))
     : {}
+  if (renderedReadmes[RENDERED_READMES_ENVIRONMENT_KEY] === createReadmeRendererEnvironmentHash()) {
+    console.warn('[eleventy] Using pre-rendered READMEs from readmes_rendered.json')
+  } else {
+    const renderedReadmesStale = Object.keys(renderedReadmes).length > 0
+    console.warn(
+      '[eleventy] All READMEs are fetched and rendered live in the browser. '
+      + 'Tip: run `node render_readmes.mjs -i readmes.json -o readmes_rendered.json` '
+      + `to prerender READMEs${renderedReadmesStale ? ' again' : ''}.`,
+    )
+    renderedReadmes = {}
+  }
   let all_packages = util.simplifyPackageLabels(
     // eslint-disable-next-line no-unused-vars
     Object.entries(workspace.packages).map(([id, pkg]) => pkg),
@@ -598,7 +613,7 @@ export default async function (eleventyConfig) {
       installed: stat?.installs?.totals ?? 0,
       installs_window: stat?.installs?.yearly?.reduce((a, b) => a + b, 0) ?? 0,
       ...(readme_url !== pkg.readme ? { readme_url } : {}),
-      ...(renderedReadmes[pkg.readme] ? { rendered_readme: renderedReadmes[pkg.readme] } : {}),
+      ...(renderedReadmes[pkg.readme] ? { rendered_readme: renderedReadmes[pkg.readme][1] } : {}),
       ...(source_url !== pkg.source ? { source_url } : {}),
     }
   }

render_readmes.mjs

@@ -5,29 +5,64 @@ import { JSDOM } from 'jsdom'
 import createDOMPurify from 'dompurify'
 import { marked } from 'marked'
 import { configureMarked, renderReadme } from './static/readme-renderer.mjs'
+import {
+  RENDERED_READMES_ENVIRONMENT_KEY,
+  createReadmeRendererEnvironmentHash,
+  createReadmeSourceHash,
+} from './util/readme-render-cache.mjs'
 
 const args = parseArgs(process.argv.slice(2))
 const rawReadmes = readJson(args.input)
 const dom = new JSDOM('')
 const DOMPurify = createDOMPurify(dom.window)
-const parser = new dom.window.DOMParser()
-const renderedReadmes = {}
+
+let oldRenderedReadmes = readJson(args.output)
+const renderedReadmesEnvironment = createReadmeRendererEnvironmentHash()
+if (oldRenderedReadmes[RENDERED_READMES_ENVIRONMENT_KEY] !== renderedReadmesEnvironment) {
+  oldRenderedReadmes = {}
+}
+const renderedReadmes = {
+  [RENDERED_READMES_ENVIRONMENT_KEY]: renderedReadmesEnvironment,
+}
 
 configureMarked(marked)
 
 for (const [url, text] of Object.entries(rawReadmes)) {
+  const hash = createReadmeSourceHash(text)
+  if (oldRenderedReadmes[url] && oldRenderedReadmes[url][0] === hash) {
+    renderedReadmes[url] = [hash, oldRenderedReadmes[url][1]]
+    continue
+  }
+
   const html = renderReadme(marked, text, url, {
-    sanitize: html => DOMPurify.sanitize(html),
-    parseHtml: html => parser.parseFromString(html, 'text/html'),
+    sanitize: sanitizeHtml,
+    parseHtml,
   })
-  renderedReadmes[url] = html
+  renderedReadmes[url] = [hash, html]
 }
 
 fs.writeFileSync(args.output, JSON.stringify(renderedReadmes, null, 2) + '\n')
-console.log(`Rendered ${Object.keys(renderedReadmes).length} README files to ${args.output}`)
+console.log(`Rendered ${Object.keys(rawReadmes).length} README files to ${args.output}`)
 
 dom.window.close()
 
+function parseHtml(html) {
+  const template = dom.window.document.createElement('template')
+  template.innerHTML = html
+
+  return {
+    body: template,
+    querySelectorAll: selector => template.content.querySelectorAll(selector),
+  }
+}
+
+function sanitizeHtml(html) {
+  const fragment = DOMPurify.sanitize(html, { RETURN_DOM_FRAGMENT: true })
+  const template = dom.window.document.createElement('template')
+  template.content.append(fragment)
+  return template.innerHTML
+}
+
 function parseArgs(argv) {
   let input = 'readmes.json'
   let output = 'readmes_rendered.json'

util/readme-render-cache.mjs

@@ -0,0 +1,26 @@
+import fs from 'fs'
+import { createHash } from 'crypto'
+
+export const RENDERED_READMES_ENVIRONMENT_KEY = '__environment'
+
+export function createReadmeRendererEnvironmentHash() {
+  const hash = createHash('sha256')
+  const files = [
+    ['render_readmes.mjs', new URL('../render_readmes.mjs', import.meta.url)],
+    ['static/readme-renderer.mjs', new URL('../static/readme-renderer.mjs', import.meta.url)],
+    ['util/readme-render-cache.mjs', new URL(import.meta.url)],
+    ['package-lock.json', new URL('../package-lock.json', import.meta.url)],
+  ]
+
+  for (const [name, url] of files) {
+    hash.update(`${name}\0`)
+    hash.update(fs.readFileSync(url))
+    hash.update('\0')
+  }
+
+  return hash.digest('base64')
+}
+
+export function createReadmeSourceHash(text) {
+  return createHash('sha256').update(text).digest('base64')
+}