diff --git a/src/commands/root.ts b/src/commands/root.ts index d0f391d..8ca9b45 100644 --- a/src/commands/root.ts +++ b/src/commands/root.ts @@ -50,7 +50,8 @@ extract (selector — CSS, structured): attributes; empty sel = the match) --table → rows keyed by headers --text | --attr | --html simpler per-match output - --md readable page content as markdown (for reading docs) + --md readable page content as markdown (for reading docs; + capped at ~2000 tokens unless --all or --budget T) --where filter rows: price > 100 && name ~ /^foo/i (no eval; \`col name\` for headers with spaces) @@ -939,7 +940,12 @@ export async function root(argv: string[]) { if (flags.md) { const md = toMarkdown((document.querySelector('html') ?? document) as unknown as Element) - return emitLines(md.split('\n'), { ...opts, budget: opts.budget || 2000 }) + // --md carries a default token budget on top of --limit, because markdown + // lines are cheap individually but a whole page adds up. It is only a + // *default*: --all means all (never a truncation note pointing at the flag + // that is already set), and an explicit --budget wins. + const budget = opts.all || opts.budget > 0 ? opts.budget : 2000 + return emitLines(md.split('\n'), { ...opts, budget }) } if (flags.outline) { diff --git a/test/cli.test.ts b/test/cli.test.ts index 3b3583b..b498ee2 100644 --- a/test/cli.test.ts +++ b/test/cli.test.ts @@ -610,6 +610,45 @@ test('cap: --budget truncation resumes exactly where the note said', () => { expect([...first.out.split('\n'), ...rest.out.split('\n')]).toEqual(all) }) +test('cap: --md --all emits the whole page, no default budget', () => { + writeFileSync( + join(dir, 'long-md.html'), + `
${Array.from( + { length: 300 }, + (_, i) => `

paragraph number ${i} with enough filler text to spend a few tokens

` + ).join('')}
` + ) + const capped = ax(['long-md.html', '--md']) + expect(capped.err).toContain('more result(s) hidden') + + const all = ax(['long-md.html', '--md', '--all']) + expect(all.code).toBe(0) + expect(all.err).not.toContain('hidden') + expect(all.out.match(/paragraph number/g)).toHaveLength(300) + + // an explicit --budget still wins, even together with --all + const budgeted = ax(['long-md.html', '--md', '--all', '--budget', '200']) + expect(budgeted.err).toContain('more result(s) hidden') + expect(budgeted.out.length).toBeLessThan(all.out.length) +}) + +test('cap: --md alone stays capped by the default token budget', () => { + // Long-prose page: few enough lines that --limit 50 never fires, so what + // truncates here can only be the ~2000-token default budget. + writeFileSync( + join(dir, 'prose-md.html'), + `
${Array.from( + { length: 40 }, + (_, i) => `

paragraph ${i}: ${'filler words to pad this paragraph out. '.repeat(10)}

` + ).join('')}
` + ) + const r = ax(['prose-md.html', '--md']) + expect(r.code).toBe(0) + expect(r.err).toContain('more result(s) hidden') + expect(Number(r.err.match(/continue with --offset (\d+)/)![1]!)).toBeLessThan(50) + expect(r.out.length).toBeLessThan(2000 * 4 + 500) +}) + test('cap: --offset applies to JSON rows too', () => { const r = ax(['page.html', '.card', '--row', 'title=a', '--json', '--offset', '1']) expect(JSON.parse(r.out)).toEqual([{ title: 'Two' }])