diff --git a/README.md b/README.md index e68abd1..52e9b2f 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,9 @@ pnpm exec playwright install chromium pnpm check ``` -`pnpm check` verifies media, builds `out/`, creates its Pagefind index, checks TypeScript, and runs desktop/mobile browser tests against a static file server. There is no Next.js server in production. +`pnpm check` verifies media, builds `out/`, creates its Pagefind index and `out/search-index.json`, checks TypeScript, and runs desktop/mobile browser tests against a static file server. There is no Next.js server in production. + +`scripts/build-search-index.mjs` writes `/search-index.json` for the Core web app's command palette: one entry per visible guide with its path, title, section, description, h2/h3 anchors read from the built HTML, and frontmatter `keywords`. Set `searchable: false` in a page's frontmatter to leave it out. ## Guide map diff --git a/package.json b/package.json index c3940e1..4681d07 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "packageManager": "pnpm@10.34.5", "scripts": { "dev": "next dev", - "build": "next build && pagefind --site out --output-path out/_pagefind", + "build": "next build && pagefind --site out --output-path out/_pagefind && node scripts/build-search-index.mjs", "typecheck": "tsc --noEmit", "media:import": "node scripts/import-media.mjs", "media:enclosure": "node scripts/import-enclosure.mjs", diff --git a/scripts/build-search-index.mjs b/scripts/build-search-index.mjs new file mode 100644 index 0000000..82e3f5f --- /dev/null +++ b/scripts/build-search-index.mjs @@ -0,0 +1,161 @@ +import { createRequire } from 'node:module' +import { readFile, readdir, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { gzipSync } from 'node:zlib' + +// Writes out/search-index.json for the Core command palette. Titles and anchors +// come from the built HTML so they match what Nextra renders; descriptions and +// keywords come from the MDX frontmatter. +const require = createRequire(import.meta.url) +const { parse: parseYaml } = createRequire(require.resolve('nextra'))('yaml') +const base = 'https://sleepypod.github.io' + +const loadMeta = async (dir) => { + const src = await readFile(join(dir, '_meta.ts'), 'utf8').catch(() => null) + if (!src) return {} + const url = 'data:text/javascript;base64,' + Buffer.from(src).toString('base64') + return (await import(url)).default +} +const metaTitle = (entry) => (typeof entry === 'string' ? entry : entry?.title) +const metaHidden = (entry) => typeof entry === 'object' && entry?.display === 'hidden' + +const decode = (s) => + s + .replace(/&#x([0-9a-f]+);/gi, (_, h) => String.fromCodePoint(parseInt(h, 16))) + .replace(/&#(\d+);/g, (_, d) => String.fromCodePoint(Number(d))) + .replace(/"/g, '"') + .replace(/</g, '<') + .replace(/>/g, '>') + .replace(/ /g, ' ') + .replace(/&/g, '&') +const text = (html) => + decode(html.replace(/<[^>]*>/g, '')) + .replace(/\s+/g, ' ') + .trim() + +const cap = (s, max = 200) => { + if (s.length <= max) return s + const cut = s.slice(0, max + 1).replace(/\s+\S*$/, '') + return cut.replace(/[\s,;:.–—-]+$/, '') + '…' +} + +function plain(md) { + return md + .replace(/!\[[^\]]*\]\([^)]*\)/g, '') + .replace(/\[\^[^\]]+\]/g, '') + .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1') + .replace(/\[([^\]]*)\]\[[^\]]*\]/g, '$1') + .replace(/<[^>]+>/g, '') + .replace(/\{\/\*.*?\*\/\}/g, '') + .replace(/`([^`]*)`/g, '$1') + .replace(/(\*\*|__)(.+?)\1/g, '$2') + .replace(/(^|\W)[*_](\S(?:.*?\S)?)[*_](?=\W|$)/g, '$1$2') + .replace(/\s+/g, ' ') + .trim() +} + +function firstParagraph(body) { + const lines = body.split('\n') + let closing = null + let fence = null + let para = [] + for (const line of lines) { + if (fence) { + if (line.trimStart().startsWith(fence)) fence = null + continue + } + if (closing) { + if (line.includes(closing)) closing = null + continue + } + if (!line.trim()) { + if (para.length) break + continue + } + if (para.length) { + para.push(line) + continue + } + const fenceMatch = line.match(/^\s*(```|~~~)/) + if (fenceMatch) { + fence = fenceMatch[1] + continue + } + const tag = line.match(/^<([A-Za-z][\w.]*)/) + if (tag) { + const opened = line.slice(line.indexOf('<')) + if (!/\/>\s*$/.test(opened) && !opened.includes(``)) closing = `` + continue + } + if (/^(import|export)\s/.test(line) || /^\s|^[#>|!:-]|^\d+\.\s|^\{/.test(line)) continue + para.push(line) + } + return plain(para.join(' ')) +} + +function headings(html) { + const out = [] + // Markdown headings carry Nextra's permalink; component headings (Tabs, showcases) do not. + for (const m of html.matchAll(/]*)>([\s\S]*?)<\/h\1>/g)) { + const id = m[2].match(/\bid="([^"]+)"/)?.[1] + if (!id || !m[3].includes('subheading-anchor')) continue + out.push({ title: text(m[3]), anchor: decode(id) }) + } + return out +} + +async function walk(dir, route, section, skipped) { + const meta = await loadMeta(dir) + const pages = [] + for (const e of (await readdir(dir, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name), + )) { + const key = e.name.replace(/\.mdx$/, '') + const path = route + (key === 'index' ? '' : key + '/') + if (metaHidden(meta[key])) { + skipped.push(path) + continue + } + if (e.isDirectory()) { + const sub = section === undefined ? (metaTitle(meta[key]) ?? key) : section + pages.push(...(await walk(join(dir, e.name), path, sub, skipped))) + continue + } + if (!e.name.endsWith('.mdx')) continue + const src = await readFile(join(dir, e.name), 'utf8') + const fm = src.match(/^---\r?\n([\s\S]*?)\r?\n---\r?\n/) + const data = (fm && parseYaml(fm[1])) || {} + if (data.searchable === false || data.draft === true) { + skipped.push(path) + continue + } + const html = await readFile(join('out', path, 'index.html'), 'utf8') + const main = html.match(/]*data-pagefind-body[\s\S]*?<\/main>/)?.[0] + if (!main) throw new Error(`No indexed content in out${path}index.html`) + const h1 = main.match(/]*>([\s\S]*?)<\/h1>/)?.[1] + const keywords = Array.isArray(data.keywords) ? data.keywords.join(' ') : data.keywords + pages.push({ + path, + title: data.title || (h1 && text(h1)) || metaTitle(meta[key]) || key, + section: section ?? null, + description: cap( + (data.description || firstParagraph(src.slice(fm ? fm[0].length : 0))).trim(), + ), + headings: headings(main), + keywords: keywords ? String(keywords).trim() : '', + }) + } + return pages +} + +const skipped = [] +const pages = (await walk('content', '/', undefined, skipped)).sort((a, b) => + a.path < b.path ? -1 : a.path > b.path ? 1 : 0, +) +if (!pages.length) throw new Error('Search index has no pages') +const json = JSON.stringify({ version: 1, generatedAt: new Date().toISOString(), base, pages }) +await writeFile('out/search-index.json', json) +console.log( + `Search index: ${pages.length} pages, ${json.length} bytes (${gzipSync(json).length} gzipped)` + + (skipped.length ? `; skipped ${skipped.join(', ')}` : ''), +) diff --git a/tests/site.spec.ts b/tests/site.spec.ts index c10cf59..534d033 100644 --- a/tests/site.spec.ts +++ b/tests/site.spec.ts @@ -73,6 +73,28 @@ test('search finds an indexed product guide', async ({ page }) => { await page.keyboard.press('Enter') await expect(page).toHaveURL(/core\/(integrations|settings\/homekit)/) }) +test('docs search index paths and anchors resolve', async ({ page, request }) => { + test.setTimeout(120000) + const response = await request.get('/search-index.json') + expect(response.status()).toBe(200) + const index: { + version: number + base: string + pages: { path: string; headings: { anchor: string }[] }[] + } = await response.json() + expect(index.version).toBe(1) + expect(index.base).toBe('https://sleepypod.github.io') + expect(index.pages.length).toBeGreaterThan(0) + expect(index.pages.map((p) => p.path)).toContain('/getting-started/') + for (const { path: route, headings } of index.pages) { + expect((await page.goto(route))?.status(), route).toBe(200) + const missing = await page.evaluate( + (ids) => ids.filter((id) => !document.getElementById(id)), + headings.map((h) => h.anchor), + ) + expect(missing, route).toEqual([]) + } +}) test('videos decode and play to the end', async ({ page }) => { test.setTimeout(150000) for (const route of [