diff --git a/.github/workflows/publish_after_push.yml b/.github/workflows/publish_after_push.yml index f0fa17216..4eb3f54e1 100644 --- a/.github/workflows/publish_after_push.yml +++ b/.github/workflows/publish_after_push.yml @@ -27,6 +27,14 @@ jobs: with: node-version: "22" + - name: Restore rendered README cache + uses: actions/cache@v4 + with: + path: readmes_rendered.json + key: readmes-rendered-${{ github.run_id }} + restore-keys: | + readmes-rendered- + - name: Build run: make build diff --git a/eleventy.config.mjs b/eleventy.config.mjs index 731a27ed0..1b618d829 100644 --- a/eleventy.config.mjs +++ b/eleventy.config.mjs @@ -6,6 +6,10 @@ import * as esbuild from 'esbuild' import * as util from './eleventy.util.mjs' import * as filters from './eleventy.filters.mjs' import { bundleCss } from './util/bundle-css.mjs' +import { + RENDERED_READMES_ENVIRONMENT_KEY, + createReadmeRendererEnvironmentHash, +} from './util/readme-render-cache.mjs' const repackagerSite = 'https://repackager.sublimetext.io' const supportedRepackagerHosts = [ @@ -439,9 +443,20 @@ export default async function (eleventyConfig) { const workspace = JSON.parse(fs.readFileSync('workspace.json', 'utf8')) const stats = JSON.parse(fs.readFileSync('stats.json', 'utf8')) - const renderedReadmes = fs.existsSync('readmes_rendered.json') + let renderedReadmes = fs.existsSync('readmes_rendered.json') ? JSON.parse(fs.readFileSync('readmes_rendered.json', 'utf8')) : {} + if (renderedReadmes[RENDERED_READMES_ENVIRONMENT_KEY] === createReadmeRendererEnvironmentHash()) { + console.warn('[eleventy] Using pre-rendered READMEs from readmes_rendered.json') + } else { + const renderedReadmesStale = Object.keys(renderedReadmes).length > 0 + console.warn( + '[eleventy] All READMEs are fetched and rendered live in the browser. ' + + 'Tip: run `node render_readmes.mjs -i readmes.json -o readmes_rendered.json` ' + + `to prerender READMEs${renderedReadmesStale ? ' again' : ''}.`, + ) + renderedReadmes = {} + } let all_packages = util.simplifyPackageLabels( // eslint-disable-next-line no-unused-vars Object.entries(workspace.packages).map(([id, pkg]) => pkg), @@ -598,7 +613,7 @@ export default async function (eleventyConfig) { installed: stat?.installs?.totals ?? 0, installs_window: stat?.installs?.yearly?.reduce((a, b) => a + b, 0) ?? 0, ...(readme_url !== pkg.readme ? { readme_url } : {}), - ...(renderedReadmes[pkg.readme] ? { rendered_readme: renderedReadmes[pkg.readme] } : {}), + ...(renderedReadmes[pkg.readme] ? { rendered_readme: renderedReadmes[pkg.readme][1] } : {}), ...(source_url !== pkg.source ? { source_url } : {}), } } diff --git a/render_readmes.mjs b/render_readmes.mjs index 4716c177c..bdf65209c 100644 --- a/render_readmes.mjs +++ b/render_readmes.mjs @@ -5,29 +5,64 @@ import { JSDOM } from 'jsdom' import createDOMPurify from 'dompurify' import { marked } from 'marked' import { configureMarked, renderReadme } from './static/readme-renderer.mjs' +import { + RENDERED_READMES_ENVIRONMENT_KEY, + createReadmeRendererEnvironmentHash, + createReadmeSourceHash, +} from './util/readme-render-cache.mjs' const args = parseArgs(process.argv.slice(2)) const rawReadmes = readJson(args.input) const dom = new JSDOM('') const DOMPurify = createDOMPurify(dom.window) -const parser = new dom.window.DOMParser() -const renderedReadmes = {} + +let oldRenderedReadmes = readJson(args.output) +const renderedReadmesEnvironment = createReadmeRendererEnvironmentHash() +if (oldRenderedReadmes[RENDERED_READMES_ENVIRONMENT_KEY] !== renderedReadmesEnvironment) { + oldRenderedReadmes = {} +} +const renderedReadmes = { + [RENDERED_READMES_ENVIRONMENT_KEY]: renderedReadmesEnvironment, +} configureMarked(marked) for (const [url, text] of Object.entries(rawReadmes)) { + const hash = createReadmeSourceHash(text) + if (oldRenderedReadmes[url] && oldRenderedReadmes[url][0] === hash) { + renderedReadmes[url] = [hash, oldRenderedReadmes[url][1]] + continue + } + const html = renderReadme(marked, text, url, { - sanitize: html => DOMPurify.sanitize(html), - parseHtml: html => parser.parseFromString(html, 'text/html'), + sanitize: sanitizeHtml, + parseHtml, }) - renderedReadmes[url] = html + renderedReadmes[url] = [hash, html] } fs.writeFileSync(args.output, JSON.stringify(renderedReadmes, null, 2) + '\n') -console.log(`Rendered ${Object.keys(renderedReadmes).length} README files to ${args.output}`) +console.log(`Rendered ${Object.keys(rawReadmes).length} README files to ${args.output}`) dom.window.close() +function parseHtml(html) { + const template = dom.window.document.createElement('template') + template.innerHTML = html + + return { + body: template, + querySelectorAll: selector => template.content.querySelectorAll(selector), + } +} + +function sanitizeHtml(html) { + const fragment = DOMPurify.sanitize(html, { RETURN_DOM_FRAGMENT: true }) + const template = dom.window.document.createElement('template') + template.content.append(fragment) + return template.innerHTML +} + function parseArgs(argv) { let input = 'readmes.json' let output = 'readmes_rendered.json' diff --git a/util/readme-render-cache.mjs b/util/readme-render-cache.mjs new file mode 100644 index 000000000..cff215886 --- /dev/null +++ b/util/readme-render-cache.mjs @@ -0,0 +1,26 @@ +import fs from 'fs' +import { createHash } from 'crypto' + +export const RENDERED_READMES_ENVIRONMENT_KEY = '__environment' + +export function createReadmeRendererEnvironmentHash() { + const hash = createHash('sha256') + const files = [ + ['render_readmes.mjs', new URL('../render_readmes.mjs', import.meta.url)], + ['static/readme-renderer.mjs', new URL('../static/readme-renderer.mjs', import.meta.url)], + ['util/readme-render-cache.mjs', new URL(import.meta.url)], + ['package-lock.json', new URL('../package-lock.json', import.meta.url)], + ] + + for (const [name, url] of files) { + hash.update(`${name}\0`) + hash.update(fs.readFileSync(url)) + hash.update('\0') + } + + return hash.digest('base64') +} + +export function createReadmeSourceHash(text) { + return createHash('sha256').update(text).digest('base64') +}