From fb9e421481336039c00b9ed11cfd4eed7a8859e4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 27 Aug 2026 17:01:01 +0000 Subject: [PATCH 1/2] fix(import): keep Fern/Mintlify/Archbee nav when llms.txt is larger The 75% coverage gate was written for thin HTML scrapes (one tab of a multi-tab site). After #41 it also ran against canonical sidebars, so a curated Fern/Mintlify/Archbee tree that listed a subset of llms.txt was discarded and the importer fell back to invented llms.txt clusters. Skip that gate for those probes. Orphans still slot by path afterwards. Co-authored-by: Jon Ursenbach --- src/commands/import.js | 57 ++++++++++++++++++++----------- src/commands/import.test.js | 67 +++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 19 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index 80020e6..d5f1c64 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -605,27 +605,26 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna if (debugSnapshots) { debugSnapshots[`02-scraped-raw${dbgSuffix}.json`] = scraped ? JSON.parse(JSON.stringify(scraped)) : null } - // If the sidebar scrape covers less than 75% of the llms.txt URLs, the - // scrape is too thin to trust as the import's spine. Common on multi-tab - // docs (Stripe, AWS, Twilio, Xata) where each page only renders its own - // tab's sidebar — the visible categories would otherwise absorb hundreds - // of orphan URLs via prefix-matching and produce a misleading tree (e.g. - // every /docs/* URL dumped under a single "Overview > Xata Documentation" - // node because that's the one /docs page the scrape saw). Discard the - // scrape and fall through to the llms.txt path, which uses URL-based - // clustering when multiple files were merged. - // API-reference pages are excluded from the denominator: sidebars routinely - // omit generated endpoint stubs, and those pages are swept into reference/ - // regardless of nav quality, so they say nothing about the nav's fitness as - // the import's spine. + // HTML sidebar scrapes that cover <75% of llms.txt URLs are too thin to + // trust as the import's spine. Common on multi-tab docs (Stripe, AWS, + // Twilio, Xata) where each page only renders its own tab's sidebar — the + // visible categories would otherwise absorb hundreds of orphan URLs via + // prefix-matching and produce a misleading tree. Discard that scrape and + // fall through to the llms.txt path. + // + // Mintlify / Fern / Archbee navs are the authored sidebar, not a scrape. + // They routinely list a curated subset of what llms.txt enumerates + // (hidden pages, legacy paths, extra API stubs). Applying the same + // threshold would throw away the canonical tree we just recovered and + // invent categories from llms.txt — the failure mode those probes exist + // to prevent. Orphans still get slotted by path below. + const canonicalNav = !!(mintlifyNav || fernNav || archbeeNav) let scrapeDiscardedForCoverage = false - if (scraped && llms && knownUrls.length > 0) { - const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) - const nonReferenceKnown = knownUrls.filter((p) => !urlIsApiReference(p.url)) - const coverage = nonReferenceKnown.length > 0 ? scrapedPages / nonReferenceKnown.length : 1 - if (coverage < 0.75) { + if (scraped && llms && knownUrls.length > 0 && !canonicalNav) { + const coverage = htmlScrapeCoverage(scraped, knownUrls) + if (coverage.ratio < 0.75) { styles.info( - `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%${nonReferenceKnown.length < knownUrls.length ? `, ${knownUrls.length - nonReferenceKnown.length} api-reference pages excluded` : ''}) — discarding scrape and organizing from llms.txt.`, + `Scrape covered ${styles.bold(Math.round(coverage.ratio * 100) + '%')} of llms.txt pages (need ≥75%${coverage.excludedApiReference > 0 ? `, ${coverage.excludedApiReference} api-reference pages excluded` : ''}) — discarding scrape and organizing from llms.txt.`, ) scraped = null scrapeDiscardedForCoverage = true @@ -2646,6 +2645,25 @@ function urlIsApiReference(url) { } } +/** + * How completely an HTML-scraped nav covers the llms.txt URL list. + * API-reference pages are excluded from the denominator: sidebars routinely + * omit generated endpoint stubs, and those pages are swept into reference/ + * regardless of nav quality, so they say nothing about the nav's fitness as + * the import's spine. + */ +function htmlScrapeCoverage(scraped, knownUrls) { + const scrapedPages = (scraped?.categories || []).reduce((n, c) => n + countUrlPagesDeep(c.pages), 0) + const nonReferenceKnown = (knownUrls || []).filter((p) => !urlIsApiReference(p.url)) + const ratio = nonReferenceKnown.length > 0 ? scrapedPages / nonReferenceKnown.length : 1 + return { + ratio, + scrapedPages, + nonReferenceKnown: nonReferenceKnown.length, + excludedApiReference: (knownUrls || []).length - nonReferenceKnown.length, + } +} + function reclassifyReferencePages(scraped) { return reclassifyPagesByUrlSegment(scraped, { segmentRe: API_REFERENCE_URL_SEGMENT_RE, @@ -5319,6 +5337,7 @@ export const __test__ = { sitemapUrlsToKnownUrls, extractOasSpecUrlsFromParsed, downloadOasSpecs, + htmlScrapeCoverage, } function formatDuration(ms) { diff --git a/src/commands/import.test.js b/src/commands/import.test.js index fc2aaa2..2e65291 100644 --- a/src/commands/import.test.js +++ b/src/commands/import.test.js @@ -380,3 +380,70 @@ test('tryFernNav keeps the entry tree when a tab fetch fails', async () => { const nav = await __test__.tryFernNav('https://fern.example/intro', []) assertGetStartedTree(nav) }) + +function pages(...urls) { + return urls.map((url) => ({ title: url, url })) +} + +test('htmlScrapeCoverage drops below 75% when a thin HTML scrape sees a fraction of llms.txt', () => { + const scraped = { categories: [{ title: 'Docs', pages: pages('https://ex.com/docs/a', 'https://ex.com/docs/b') }] } + const known = pages( + 'https://ex.com/docs/a', + 'https://ex.com/docs/b', + 'https://ex.com/docs/c', + 'https://ex.com/docs/d', + 'https://ex.com/docs/e', + ) + const coverage = __test__.htmlScrapeCoverage(scraped, known) + assert.equal(coverage.scrapedPages, 2) + assert.equal(coverage.nonReferenceKnown, 5) + assert.ok(coverage.ratio < 0.75) +}) + +test('htmlScrapeCoverage stays at or above 75% when the scrape covers most llms.txt pages', () => { + const scraped = { + categories: [{ title: 'Docs', pages: pages('https://ex.com/docs/a', 'https://ex.com/docs/b', 'https://ex.com/docs/c', 'https://ex.com/docs/d') }], + } + const known = pages( + 'https://ex.com/docs/a', + 'https://ex.com/docs/b', + 'https://ex.com/docs/c', + 'https://ex.com/docs/d', + 'https://ex.com/docs/e', + ) + const coverage = __test__.htmlScrapeCoverage(scraped, known) + assert.equal(coverage.ratio, 0.8) + assert.ok(coverage.ratio >= 0.75) +}) + +test('htmlScrapeCoverage ignores api-reference llms.txt rows so endpoint stubs do not tank a real sidebar', () => { + const scraped = { categories: [{ title: 'Docs', pages: pages('https://ex.com/docs/a') }] } + const known = pages('https://ex.com/docs/a', 'https://ex.com/api-reference/list', 'https://ex.com/endpoints/create') + const coverage = __test__.htmlScrapeCoverage(scraped, known) + assert.equal(coverage.nonReferenceKnown, 1) + assert.equal(coverage.excludedApiReference, 2) + assert.equal(coverage.ratio, 1) +}) + +test('htmlScrapeCoverage treats a curated Fern-sized sidebar against a larger llms.txt as below the HTML-scrape cutoff', () => { + // Typical Fern + llms.txt shape: authored sidebar is a subset, llms.txt + // also lists hidden/legacy/API pages. The HTML-scrape cutoff would discard + // that tree; canonical Fern/Mintlify/Archbee navs must skip this check. + const scraped = { + categories: [{ title: 'Get started', pages: pages('https://fern.example/intro', 'https://fern.example/guides', 'https://fern.example/nested/deep') }], + } + const known = pages( + 'https://fern.example/intro', + 'https://fern.example/guides', + 'https://fern.example/nested/deep', + 'https://fern.example/hidden', + 'https://fern.example/legacy', + 'https://fern.example/old-guide', + 'https://fern.example/changelog', + 'https://fern.example/blog-in-docs', + ) + const coverage = __test__.htmlScrapeCoverage(scraped, known) + assert.equal(coverage.scrapedPages, 3) + assert.equal(coverage.nonReferenceKnown, 8) + assert.ok(coverage.ratio < 0.75) +}) From e69eb17ac646b6da7e54c962ccee77b76ef7f53b Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 27 Aug 2026 17:07:29 +0000 Subject: [PATCH 2/2] fix(import): keep canonical nav through the orphan-ratio recluster The 75% coverage exemption still handed Fern/Mintlify/Archbee trees to the existing 2x-orphan gate, which replaced the authored sidebar with URL-derived categories whenever llms.txt listed enough extra pages. Skip that recluster for canonical navs too. Leftover llms.txt rows still slot by path or bucket into extra categories; the authored spine stays. Co-authored-by: Jon Ursenbach --- src/commands/import.js | 37 ++++++++++++++++++++++++++----------- src/commands/import.test.js | 13 +++++++++++++ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/commands/import.js b/src/commands/import.js index d5f1c64..118e2db 100644 --- a/src/commands/import.js +++ b/src/commands/import.js @@ -614,10 +614,11 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna // // Mintlify / Fern / Archbee navs are the authored sidebar, not a scrape. // They routinely list a curated subset of what llms.txt enumerates - // (hidden pages, legacy paths, extra API stubs). Applying the same - // threshold would throw away the canonical tree we just recovered and - // invent categories from llms.txt — the failure mode those probes exist - // to prevent. Orphans still get slotted by path below. + // (hidden pages, legacy paths, extra API stubs). Both thin-scrape gates + // (this coverage check, and the orphan-ratio recluster below) would + // throw away that tree and invent categories from llms.txt — the failure + // mode those probes exist to prevent. Orphans still get slotted by path + // and leftover ones are bucketed; the authored categories stay. const canonicalNav = !!(mintlifyNav || fernNav || archbeeNav) let scrapeDiscardedForCoverage = false if (scraped && llms && knownUrls.length > 0 && !canonicalNav) { @@ -664,18 +665,19 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } if (slotted.length > 0) { - // When orphans dwarf direct matches, the sidebar scrape was too thin + // When orphans dwarf direct matches, an HTML scrape was too thin // to trust as the import's spine — keeping it would produce a small // "real" tree plus a soup of bucketed-by-URL-type orphan categories. // Discard the scrape and cluster every page (scrape + orphans) by // its top URL segment instead, so each `/docs/`, `/sdk/`, `/rest-api/` // becomes its own category. `nestByUrlHierarchy` (later) handles the // empty-parent nesting within each category. - - // Trip when orphans are at least 2× the direct matches: the scrape - // accounts for less than a third of the known pages, so its category - // labels aren't a trustworthy spine for the remainder. - const orphansDwarfDirect = slotted.length >= directMatches * 2 + // + // Canonical Mintlify/Fern/Archbee trees skip this: extra llms.txt + // rows are expected, and replacing the authored categories with + // URL clusters (then leaving them flat because Fern suppresses + // nestByUrlHierarchy) is the bug those probes exist to prevent. + const orphansDwarfDirect = orphansDwarfHtmlScrape(slotted.length, directMatches, { canonical: canonicalNav }) const scrapeAllPages = scraped.categories.flatMap((c) => collectUrlPagesDeep(c.pages)) let reclustered = null if (orphansDwarfDirect) { @@ -683,7 +685,7 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna } styles.info( styles.dim( - ` orphan triage: ${slotted.length} orphan${slotted.length === 1 ? '' : 's'} vs ${directMatches} direct match${directMatches === 1 ? '' : 'es'} (ratio ${directMatches === 0 ? '∞' : (slotted.length / directMatches).toFixed(2)}) — gate ${orphansDwarfDirect ? 'tripped' : 'NOT tripped'} (need ≥2.00); URL re-cluster ${reclustered ? `→ ${reclustered.length} categor${reclustered.length === 1 ? 'y' : 'ies'}` : 'skipped'}`, + ` orphan triage: ${slotted.length} orphan${slotted.length === 1 ? '' : 's'} vs ${directMatches} direct match${directMatches === 1 ? '' : 'es'} (ratio ${directMatches === 0 ? '∞' : (slotted.length / directMatches).toFixed(2)}) — gate ${orphansDwarfDirect ? 'tripped' : 'NOT tripped'}${canonicalNav ? ' (canonical nav)' : ''} (need ≥2.00); URL re-cluster ${reclustered ? `→ ${reclustered.length} categor${reclustered.length === 1 ? 'y' : 'ies'}` : 'skipped'}`, ), ) if (reclustered) { @@ -2664,6 +2666,18 @@ function htmlScrapeCoverage(scraped, knownUrls) { } } +/** + * HTML scrapes whose leftover orphans are at least 2× the direct matches + * account for less than a third of known pages, so their category labels + * aren't a trustworthy spine. Canonical Mintlify/Fern/Archbee trees skip + * this — extra llms.txt rows are expected and get slotted/bucketed instead + * of replacing the authored categories. + */ +function orphansDwarfHtmlScrape(orphanCount, directMatches, { canonical } = {}) { + if (canonical) return false + return orphanCount >= directMatches * 2 +} + function reclassifyReferencePages(scraped) { return reclassifyPagesByUrlSegment(scraped, { segmentRe: API_REFERENCE_URL_SEGMENT_RE, @@ -5338,6 +5352,7 @@ export const __test__ = { extractOasSpecUrlsFromParsed, downloadOasSpecs, htmlScrapeCoverage, + orphansDwarfHtmlScrape, } function formatDuration(ms) { diff --git a/src/commands/import.test.js b/src/commands/import.test.js index 2e65291..7ca2810 100644 --- a/src/commands/import.test.js +++ b/src/commands/import.test.js @@ -425,6 +425,19 @@ test('htmlScrapeCoverage ignores api-reference llms.txt rows so endpoint stubs d assert.equal(coverage.ratio, 1) }) +test('orphansDwarfHtmlScrape trips when leftovers are at least twice the direct matches', () => { + assert.equal(__test__.orphansDwarfHtmlScrape(6, 3), true) + assert.equal(__test__.orphansDwarfHtmlScrape(5, 3), false) + assert.equal(__test__.orphansDwarfHtmlScrape(0, 4), false) +}) + +test('orphansDwarfHtmlScrape never trips for a canonical Fern/Mintlify/Archbee nav', () => { + // Same 6-orphan / 3-direct shape that would discard an HTML scrape — the + // authored tree must stay, with leftovers slotted or bucketed instead. + assert.equal(__test__.orphansDwarfHtmlScrape(6, 3, { canonical: true }), false) + assert.equal(__test__.orphansDwarfHtmlScrape(100, 1, { canonical: true }), false) +}) + test('htmlScrapeCoverage treats a curated Fern-sized sidebar against a larger llms.txt as below the HTML-scrape cutoff', () => { // Typical Fern + llms.txt shape: authored sidebar is a subset, llms.txt // also lists hidden/legacy/API pages. The HTML-scrape cutoff would discard