From 2bea902304cc2797bb1cbf7e45391969913d63df Mon Sep 17 00:00:00 2001 From: herr kaste Date: Tue, 21 Jul 2026 19:42:38 +0200 Subject: [PATCH 1/5] Reduce README renderer memory footprint and runtime Avoid the jsdom DOMParser and DOMPurify string sanitization paths in render_readmes.mjs. Those paths retained substantial DOM state during a cold README render and grew heap roughly linearly with package count. Profiling the old implementation on 4,892 READMEs showed a peak working set around 3.5 GB and a runtime around 30 seconds. The patched version uses detached template fragments for post-processing and asks DOMPurify for a DOM fragment before serializing it back to HTML. The same cold render now peaks around 550 MB and runs in about 18 seconds. That is roughly 84% less peak memory and 39% faster for the current corpus. The remaining growth is small enough that 10k packages should stay well below the default Node heap limit, instead of risking an out-of-memory failure around 6k-7k. Security behavior is preserved: DOMPurify is still the final sanitizer, with the same default policy. Regression checks rendered all 4,892 README files byte-for-byte identically to the existing readmes_rendered.json, and static/readme-renderer.test.mjs passed. --- render_readmes.mjs | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/render_readmes.mjs b/render_readmes.mjs index 4716c177c..283b14661 100644 --- a/render_readmes.mjs +++ b/render_readmes.mjs @@ -10,15 +10,14 @@ const args = parseArgs(process.argv.slice(2)) const rawReadmes = readJson(args.input) const dom = new JSDOM('') const DOMPurify = createDOMPurify(dom.window) -const parser = new dom.window.DOMParser() const renderedReadmes = {} configureMarked(marked) for (const [url, text] of Object.entries(rawReadmes)) { const html = renderReadme(marked, text, url, { - sanitize: html => DOMPurify.sanitize(html), - parseHtml: html => parser.parseFromString(html, 'text/html'), + sanitize: sanitizeHtml, + parseHtml, }) renderedReadmes[url] = html } @@ -28,6 +27,23 @@ console.log(`Rendered ${Object.keys(renderedReadmes).length} README files to ${a dom.window.close() +function parseHtml(html) { + const template = dom.window.document.createElement('template') + template.innerHTML = html + + return { + body: template, + querySelectorAll: selector => template.content.querySelectorAll(selector), + } +} + +function sanitizeHtml(html) { + const fragment = DOMPurify.sanitize(html, { RETURN_DOM_FRAGMENT: true }) + const template = dom.window.document.createElement('template') + template.content.append(fragment) + return template.innerHTML +} + function parseArgs(argv) { let input = 'readmes.json' let output = 'readmes_rendered.json' From 68b8200f9f111c94fa440c35c5b644d1a464c813 Mon Sep 17 00:00:00 2001 From: Benjamin Schaaf Date: Tue, 21 Jul 2026 13:16:57 +1000 Subject: [PATCH 2/5] Avoid re-rendering readmes The readme rendering is slow and memory intensive, but there's no reason to redo all that work every time. We now store a hash of the text used to render the readme inside readmes_renderes.json to avoid re-rendering. --- eleventy.config.mjs | 2 +- render_readmes.mjs | 11 ++++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/eleventy.config.mjs b/eleventy.config.mjs index 731a27ed0..6d654785e 100644 --- a/eleventy.config.mjs +++ b/eleventy.config.mjs @@ -598,7 +598,7 @@ export default async function (eleventyConfig) { installed: stat?.installs?.totals ?? 0, installs_window: stat?.installs?.yearly?.reduce((a, b) => a + b, 0) ?? 0, ...(readme_url !== pkg.readme ? { readme_url } : {}), - ...(renderedReadmes[pkg.readme] ? { rendered_readme: renderedReadmes[pkg.readme] } : {}), + ...(renderedReadmes[pkg.readme] ? { rendered_readme: renderedReadmes[pkg.readme][1] } : {}), ...(source_url !== pkg.source ? { source_url } : {}), } } diff --git a/render_readmes.mjs b/render_readmes.mjs index 283b14661..da00a97f3 100644 --- a/render_readmes.mjs +++ b/render_readmes.mjs @@ -5,21 +5,30 @@ import { JSDOM } from 'jsdom' import createDOMPurify from 'dompurify' import { marked } from 'marked' import { configureMarked, renderReadme } from './static/readme-renderer.mjs' +import { createHash } from 'crypto' const args = parseArgs(process.argv.slice(2)) const rawReadmes = readJson(args.input) const dom = new JSDOM('') const DOMPurify = createDOMPurify(dom.window) + +const oldRenderedReadmes = readJson(args.output) const renderedReadmes = {} configureMarked(marked) for (const [url, text] of Object.entries(rawReadmes)) { + const hash = createHash('sha256').update(text).digest('base64') + if (oldRenderedReadmes[url] && oldRenderedReadmes[url][0] === hash) { + renderedReadmes[url] = [hash, oldRenderedReadmes[url][1]] + continue + } + const html = renderReadme(marked, text, url, { sanitize: sanitizeHtml, parseHtml, }) - renderedReadmes[url] = html + renderedReadmes[url] = [hash, html] } fs.writeFileSync(args.output, JSON.stringify(renderedReadmes, null, 2) + '\n') From f0a46030a52bc16370040c52bf36011d5c5e6eb9 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Tue, 21 Jul 2026 20:31:36 +0200 Subject: [PATCH 3/5] Also record README renderer environment for the cache Keep the per-README cache hash focused on the source README text, and record the renderer context once in readmes_rendered.json as the __environment entry. The renderer environment hash includes render_readmes.mjs, the shared README renderer, this cache helper, and package-lock.json. If any of those inputs change, the prerender file is treated as coming from a changed rendering environment and all entries are re-computed. --- render_readmes.mjs | 20 +++++++++++++++----- util/readme-render-cache.mjs | 26 ++++++++++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) create mode 100644 util/readme-render-cache.mjs diff --git a/render_readmes.mjs b/render_readmes.mjs index da00a97f3..bdf65209c 100644 --- a/render_readmes.mjs +++ b/render_readmes.mjs @@ -5,20 +5,30 @@ import { JSDOM } from 'jsdom' import createDOMPurify from 'dompurify' import { marked } from 'marked' import { configureMarked, renderReadme } from './static/readme-renderer.mjs' -import { createHash } from 'crypto' +import { + RENDERED_READMES_ENVIRONMENT_KEY, + createReadmeRendererEnvironmentHash, + createReadmeSourceHash, +} from './util/readme-render-cache.mjs' const args = parseArgs(process.argv.slice(2)) const rawReadmes = readJson(args.input) const dom = new JSDOM('') const DOMPurify = createDOMPurify(dom.window) -const oldRenderedReadmes = readJson(args.output) -const renderedReadmes = {} +let oldRenderedReadmes = readJson(args.output) +const renderedReadmesEnvironment = createReadmeRendererEnvironmentHash() +if (oldRenderedReadmes[RENDERED_READMES_ENVIRONMENT_KEY] !== renderedReadmesEnvironment) { + oldRenderedReadmes = {} +} +const renderedReadmes = { + [RENDERED_READMES_ENVIRONMENT_KEY]: renderedReadmesEnvironment, +} configureMarked(marked) for (const [url, text] of Object.entries(rawReadmes)) { - const hash = createHash('sha256').update(text).digest('base64') + const hash = createReadmeSourceHash(text) if (oldRenderedReadmes[url] && oldRenderedReadmes[url][0] === hash) { renderedReadmes[url] = [hash, oldRenderedReadmes[url][1]] continue @@ -32,7 +42,7 @@ for (const [url, text] of Object.entries(rawReadmes)) { } fs.writeFileSync(args.output, JSON.stringify(renderedReadmes, null, 2) + '\n') -console.log(`Rendered ${Object.keys(renderedReadmes).length} README files to ${args.output}`) +console.log(`Rendered ${Object.keys(rawReadmes).length} README files to ${args.output}`) dom.window.close() diff --git a/util/readme-render-cache.mjs b/util/readme-render-cache.mjs new file mode 100644 index 000000000..cff215886 --- /dev/null +++ b/util/readme-render-cache.mjs @@ -0,0 +1,26 @@ +import fs from 'fs' +import { createHash } from 'crypto' + +export const RENDERED_READMES_ENVIRONMENT_KEY = '__environment' + +export function createReadmeRendererEnvironmentHash() { + const hash = createHash('sha256') + const files = [ + ['render_readmes.mjs', new URL('../render_readmes.mjs', import.meta.url)], + ['static/readme-renderer.mjs', new URL('../static/readme-renderer.mjs', import.meta.url)], + ['util/readme-render-cache.mjs', new URL(import.meta.url)], + ['package-lock.json', new URL('../package-lock.json', import.meta.url)], + ] + + for (const [name, url] of files) { + hash.update(`${name}\0`) + hash.update(fs.readFileSync(url)) + hash.update('\0') + } + + return hash.digest('base64') +} + +export function createReadmeSourceHash(text) { + return createHash('sha256').update(text).digest('base64') +} From 03dbd86839e18c77d09488a86e539ad0a030c1df Mon Sep 17 00:00:00 2001 From: herr kaste Date: Tue, 21 Jul 2026 20:31:52 +0200 Subject: [PATCH 4/5] Cache rendered READMEs in CI Restore readmes_rendered.json before running the build so render_readmes.mjs can reuse unchanged entries across workflow runs. Use a broad rolling Actions cache key and leave cache correctness to the script's per-entry hashes. --- .github/workflows/publish_after_push.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/publish_after_push.yml b/.github/workflows/publish_after_push.yml index f0fa17216..4eb3f54e1 100644 --- a/.github/workflows/publish_after_push.yml +++ b/.github/workflows/publish_after_push.yml @@ -27,6 +27,14 @@ jobs: with: node-version: "22" + - name: Restore rendered README cache + uses: actions/cache@v4 + with: + path: readmes_rendered.json + key: readmes-rendered-${{ github.run_id }} + restore-keys: | + readmes-rendered- + - name: Build run: make build From 2ee365288d31b41eb3a7aec75cb2bf7a69c415f4 Mon Sep 17 00:00:00 2001 From: herr kaste Date: Wed, 22 Jul 2026 12:54:50 +0200 Subject: [PATCH 5/5] Ignore stale README prerenders in builds Check the rendered README environment marker when Eleventy starts. If the prerendered file was built with a different renderer environment, discard it instead of injecting stale cached HTML into package pages. This keeps renderer hacking honest: local builds and the dev server fall back to the existing browser fetch/render path until readmes_rendered.json is regenerated with the current renderer. --- eleventy.config.mjs | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/eleventy.config.mjs b/eleventy.config.mjs index 6d654785e..1b618d829 100644 --- a/eleventy.config.mjs +++ b/eleventy.config.mjs @@ -6,6 +6,10 @@ import * as esbuild from 'esbuild' import * as util from './eleventy.util.mjs' import * as filters from './eleventy.filters.mjs' import { bundleCss } from './util/bundle-css.mjs' +import { + RENDERED_READMES_ENVIRONMENT_KEY, + createReadmeRendererEnvironmentHash, +} from './util/readme-render-cache.mjs' const repackagerSite = 'https://repackager.sublimetext.io' const supportedRepackagerHosts = [ @@ -439,9 +443,20 @@ export default async function (eleventyConfig) { const workspace = JSON.parse(fs.readFileSync('workspace.json', 'utf8')) const stats = JSON.parse(fs.readFileSync('stats.json', 'utf8')) - const renderedReadmes = fs.existsSync('readmes_rendered.json') + let renderedReadmes = fs.existsSync('readmes_rendered.json') ? JSON.parse(fs.readFileSync('readmes_rendered.json', 'utf8')) : {} + if (renderedReadmes[RENDERED_READMES_ENVIRONMENT_KEY] === createReadmeRendererEnvironmentHash()) { + console.warn('[eleventy] Using pre-rendered READMEs from readmes_rendered.json') + } else { + const renderedReadmesStale = Object.keys(renderedReadmes).length > 0 + console.warn( + '[eleventy] All READMEs are fetched and rendered live in the browser. ' + + 'Tip: run `node render_readmes.mjs -i readmes.json -o readmes_rendered.json` ' + + `to prerender READMEs${renderedReadmesStale ? ' again' : ''}.`, + ) + renderedReadmes = {} + } let all_packages = util.simplifyPackageLabels( // eslint-disable-next-line no-unused-vars Object.entries(workspace.packages).map(([id, pkg]) => pkg),