fix(import): keep Fern/Mintlify/Archbee nav when llms.txt is larger

src/commands/import.js

@@ -605,27 +605,27 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
   if (debugSnapshots) {
     debugSnapshots[`02-scraped-raw${dbgSuffix}.json`] = scraped ? JSON.parse(JSON.stringify(scraped)) : null
   }
-  // If the sidebar scrape covers less than 75% of the llms.txt URLs, the
-  // scrape is too thin to trust as the import's spine. Common on multi-tab
-  // docs (Stripe, AWS, Twilio, Xata) where each page only renders its own
-  // tab's sidebar — the visible categories would otherwise absorb hundreds
-  // of orphan URLs via prefix-matching and produce a misleading tree (e.g.
-  // every /docs/* URL dumped under a single "Overview > Xata Documentation"
-  // node because that's the one /docs page the scrape saw). Discard the
-  // scrape and fall through to the llms.txt path, which uses URL-based
-  // clustering when multiple files were merged.
-  // API-reference pages are excluded from the denominator: sidebars routinely
-  // omit generated endpoint stubs, and those pages are swept into reference/
-  // regardless of nav quality, so they say nothing about the nav's fitness as
-  // the import's spine.
+  // HTML sidebar scrapes that cover <75% of llms.txt URLs are too thin to
+  // trust as the import's spine. Common on multi-tab docs (Stripe, AWS,
+  // Twilio, Xata) where each page only renders its own tab's sidebar — the
+  // visible categories would otherwise absorb hundreds of orphan URLs via
+  // prefix-matching and produce a misleading tree. Discard that scrape and
+  // fall through to the llms.txt path.
+  //
+  // Mintlify / Fern / Archbee navs are the authored sidebar, not a scrape.
+  // They routinely list a curated subset of what llms.txt enumerates
+  // (hidden pages, legacy paths, extra API stubs). Both thin-scrape gates
+  // (this coverage check, and the orphan-ratio recluster below) would
+  // throw away that tree and invent categories from llms.txt — the failure
+  // mode those probes exist to prevent. Orphans still get slotted by path
+  // and leftover ones are bucketed; the authored categories stay.
+  const canonicalNav = !!(mintlifyNav || fernNav || archbeeNav)
   let scrapeDiscardedForCoverage = false
-  if (scraped && llms && knownUrls.length > 0) {
-    const scrapedPages = scraped.categories.reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
-    const nonReferenceKnown = knownUrls.filter((p) => !urlIsApiReference(p.url))
-    const coverage = nonReferenceKnown.length > 0 ? scrapedPages / nonReferenceKnown.length : 1
-    if (coverage < 0.75) {
+  if (scraped && llms && knownUrls.length > 0 && !canonicalNav) {
+    const coverage = htmlScrapeCoverage(scraped, knownUrls)
+    if (coverage.ratio < 0.75) {
       styles.info(
-        `Scrape covered ${styles.bold(Math.round(coverage * 100) + '%')} of llms.txt pages (need ≥75%${nonReferenceKnown.length < knownUrls.length ? `, ${knownUrls.length - nonReferenceKnown.length} api-reference pages excluded` : ''}) — discarding scrape and organizing from llms.txt.`,
+        `Scrape covered ${styles.bold(Math.round(coverage.ratio * 100) + '%')} of llms.txt pages (need ≥75%${coverage.excludedApiReference > 0 ? `, ${coverage.excludedApiReference} api-reference pages excluded` : ''}) — discarding scrape and organizing from llms.txt.`,
       )
       scraped = null
       scrapeDiscardedForCoverage = true
@@ -665,26 +665,27 @@ async function produceOrganizedForSource(sourceUrl, options, timePhase, debugSna
       }
 
       if (slotted.length > 0) {
-        // When orphans dwarf direct matches, the sidebar scrape was too thin
+        // When orphans dwarf direct matches, an HTML scrape was too thin
         // to trust as the import's spine — keeping it would produce a small
         // "real" tree plus a soup of bucketed-by-URL-type orphan categories.
         // Discard the scrape and cluster every page (scrape + orphans) by
         // its top URL segment instead, so each `/docs/`, `/sdk/`, `/rest-api/`
         // becomes its own category. `nestByUrlHierarchy` (later) handles the
         // empty-parent nesting within each category.
-
-        // Trip when orphans are at least 2× the direct matches: the scrape
-        // accounts for less than a third of the known pages, so its category
-        // labels aren't a trustworthy spine for the remainder.
-        const orphansDwarfDirect = slotted.length >= directMatches * 2
+        //
+        // Canonical Mintlify/Fern/Archbee trees skip this: extra llms.txt
+        // rows are expected, and replacing the authored categories with
+        // URL clusters (then leaving them flat because Fern suppresses
+        // nestByUrlHierarchy) is the bug those probes exist to prevent.
+        const orphansDwarfDirect = orphansDwarfHtmlScrape(slotted.length, directMatches, { canonical: canonicalNav })
         const scrapeAllPages = scraped.categories.flatMap((c) => collectUrlPagesDeep(c.pages))
         let reclustered = null
         if (orphansDwarfDirect) {
           reclustered = clusterByUrlPath([...scrapeAllPages, ...slotted])
         }
         styles.info(
           styles.dim(
-            `  orphan triage: ${slotted.length} orphan${slotted.length === 1 ? '' : 's'} vs ${directMatches} direct match${directMatches === 1 ? '' : 'es'} (ratio ${directMatches === 0 ? '∞' : (slotted.length / directMatches).toFixed(2)}) — gate ${orphansDwarfDirect ? 'tripped' : 'NOT tripped'} (need ≥2.00); URL re-cluster ${reclustered ? `→ ${reclustered.length} categor${reclustered.length === 1 ? 'y' : 'ies'}` : 'skipped'}`,
+            `  orphan triage: ${slotted.length} orphan${slotted.length === 1 ? '' : 's'} vs ${directMatches} direct match${directMatches === 1 ? '' : 'es'} (ratio ${directMatches === 0 ? '∞' : (slotted.length / directMatches).toFixed(2)}) — gate ${orphansDwarfDirect ? 'tripped' : 'NOT tripped'}${canonicalNav ? ' (canonical nav)' : ''} (need ≥2.00); URL re-cluster ${reclustered ? `→ ${reclustered.length} categor${reclustered.length === 1 ? 'y' : 'ies'}` : 'skipped'}`,
           ),
         )
         if (reclustered) {
@@ -2646,6 +2647,37 @@ function urlIsApiReference(url) {
   }
 }
 
+/**
+ * How completely an HTML-scraped nav covers the llms.txt URL list.
+ * API-reference pages are excluded from the denominator: sidebars routinely
+ * omit generated endpoint stubs, and those pages are swept into reference/
+ * regardless of nav quality, so they say nothing about the nav's fitness as
+ * the import's spine.
+ */
+function htmlScrapeCoverage(scraped, knownUrls) {
+  const scrapedPages = (scraped?.categories || []).reduce((n, c) => n + countUrlPagesDeep(c.pages), 0)
+  const nonReferenceKnown = (knownUrls || []).filter((p) => !urlIsApiReference(p.url))
+  const ratio = nonReferenceKnown.length > 0 ? scrapedPages / nonReferenceKnown.length : 1
+  return {
+    ratio,
+    scrapedPages,
+    nonReferenceKnown: nonReferenceKnown.length,
+    excludedApiReference: (knownUrls || []).length - nonReferenceKnown.length,
+  }
+}
+
+/**
+ * HTML scrapes whose leftover orphans are at least 2× the direct matches
+ * account for less than a third of known pages, so their category labels
+ * aren't a trustworthy spine. Canonical Mintlify/Fern/Archbee trees skip
+ * this — extra llms.txt rows are expected and get slotted/bucketed instead
+ * of replacing the authored categories.
+ */
+function orphansDwarfHtmlScrape(orphanCount, directMatches, { canonical } = {}) {
+  if (canonical) return false
+  return orphanCount >= directMatches * 2
+}
+
 function reclassifyReferencePages(scraped) {
   return reclassifyPagesByUrlSegment(scraped, {
     segmentRe: API_REFERENCE_URL_SEGMENT_RE,
@@ -5319,6 +5351,8 @@ export const __test__ = {
   sitemapUrlsToKnownUrls,
   extractOasSpecUrlsFromParsed,
   downloadOasSpecs,
+  htmlScrapeCoverage,
+  orphansDwarfHtmlScrape,
 }
 
 function formatDuration(ms) {

src/commands/import.test.js

@@ -380,3 +380,83 @@ test('tryFernNav keeps the entry tree when a tab fetch fails', async () => {
   const nav = await __test__.tryFernNav('https://fern.example/intro', [])
   assertGetStartedTree(nav)
 })
+
+function pages(...urls) {
+  return urls.map((url) => ({ title: url, url }))
+}
+
+test('htmlScrapeCoverage drops below 75% when a thin HTML scrape sees a fraction of llms.txt', () => {
+  const scraped = { categories: [{ title: 'Docs', pages: pages('https://ex.com/docs/a', 'https://ex.com/docs/b') }] }
+  const known = pages(
+    'https://ex.com/docs/a',
+    'https://ex.com/docs/b',
+    'https://ex.com/docs/c',
+    'https://ex.com/docs/d',
+    'https://ex.com/docs/e',
+  )
+  const coverage = __test__.htmlScrapeCoverage(scraped, known)
+  assert.equal(coverage.scrapedPages, 2)
+  assert.equal(coverage.nonReferenceKnown, 5)
+  assert.ok(coverage.ratio < 0.75)
+})
+
+test('htmlScrapeCoverage stays at or above 75% when the scrape covers most llms.txt pages', () => {
+  const scraped = {
+    categories: [{ title: 'Docs', pages: pages('https://ex.com/docs/a', 'https://ex.com/docs/b', 'https://ex.com/docs/c', 'https://ex.com/docs/d') }],
+  }
+  const known = pages(
+    'https://ex.com/docs/a',
+    'https://ex.com/docs/b',
+    'https://ex.com/docs/c',
+    'https://ex.com/docs/d',
+    'https://ex.com/docs/e',
+  )
+  const coverage = __test__.htmlScrapeCoverage(scraped, known)
+  assert.equal(coverage.ratio, 0.8)
+  assert.ok(coverage.ratio >= 0.75)
+})
+
+test('htmlScrapeCoverage ignores api-reference llms.txt rows so endpoint stubs do not tank a real sidebar', () => {
+  const scraped = { categories: [{ title: 'Docs', pages: pages('https://ex.com/docs/a') }] }
+  const known = pages('https://ex.com/docs/a', 'https://ex.com/api-reference/list', 'https://ex.com/endpoints/create')
+  const coverage = __test__.htmlScrapeCoverage(scraped, known)
+  assert.equal(coverage.nonReferenceKnown, 1)
+  assert.equal(coverage.excludedApiReference, 2)
+  assert.equal(coverage.ratio, 1)
+})
+
+test('orphansDwarfHtmlScrape trips when leftovers are at least twice the direct matches', () => {
+  assert.equal(__test__.orphansDwarfHtmlScrape(6, 3), true)
+  assert.equal(__test__.orphansDwarfHtmlScrape(5, 3), false)
+  assert.equal(__test__.orphansDwarfHtmlScrape(0, 4), false)
+})
+
+test('orphansDwarfHtmlScrape never trips for a canonical Fern/Mintlify/Archbee nav', () => {
+  // Same 6-orphan / 3-direct shape that would discard an HTML scrape — the
+  // authored tree must stay, with leftovers slotted or bucketed instead.
+  assert.equal(__test__.orphansDwarfHtmlScrape(6, 3, { canonical: true }), false)
+  assert.equal(__test__.orphansDwarfHtmlScrape(100, 1, { canonical: true }), false)
+})
+
+test('htmlScrapeCoverage treats a curated Fern-sized sidebar against a larger llms.txt as below the HTML-scrape cutoff', () => {
+  // Typical Fern + llms.txt shape: authored sidebar is a subset, llms.txt
+  // also lists hidden/legacy/API pages. The HTML-scrape cutoff would discard
+  // that tree; canonical Fern/Mintlify/Archbee navs must skip this check.
+  const scraped = {
+    categories: [{ title: 'Get started', pages: pages('https://fern.example/intro', 'https://fern.example/guides', 'https://fern.example/nested/deep') }],
+  }
+  const known = pages(
+    'https://fern.example/intro',
+    'https://fern.example/guides',
+    'https://fern.example/nested/deep',
+    'https://fern.example/hidden',
+    'https://fern.example/legacy',
+    'https://fern.example/old-guide',
+    'https://fern.example/changelog',
+    'https://fern.example/blog-in-docs',
+  )
+  const coverage = __test__.htmlScrapeCoverage(scraped, known)
+  assert.equal(coverage.scrapedPages, 3)
+  assert.equal(coverage.nonReferenceKnown, 8)
+  assert.ok(coverage.ratio < 0.75)
+})