Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/commands/root.ts
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,8 @@ extract (selector — CSS, structured):
attributes; empty sel = the match)
--table <table> → rows keyed by headers
--text | --attr <name> | --html simpler per-match output
--md readable page content as markdown (for reading docs)
--md readable page content as markdown (for reading docs;
capped at ~2000 tokens unless --all or --budget T)
--where <expr> filter rows: price > 100 && name ~ /^foo/i (no eval;
\`col name\` for headers with spaces)

Expand Down Expand Up @@ -939,7 +940,12 @@ export async function root(argv: string[]) {

if (flags.md) {
const md = toMarkdown((document.querySelector('html') ?? document) as unknown as Element)
return emitLines(md.split('\n'), { ...opts, budget: opts.budget || 2000 })
// --md carries a default token budget on top of --limit, because markdown
// lines are cheap individually but a whole page adds up. It is only a
// *default*: --all means all (never a truncation note pointing at the flag
// that is already set), and an explicit --budget wins.
const budget = opts.all || opts.budget > 0 ? opts.budget : 2000
return emitLines(md.split('\n'), { ...opts, budget })
}

if (flags.outline) {
Expand Down
39 changes: 39 additions & 0 deletions test/cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,45 @@ test('cap: --budget truncation resumes exactly where the note said', () => {
expect([...first.out.split('\n'), ...rest.out.split('\n')]).toEqual(all)
})

test('cap: --md --all emits the whole page, no default budget', () => {
writeFileSync(
join(dir, 'long-md.html'),
`<html><body><article>${Array.from(
{ length: 300 },
(_, i) => `<p>paragraph number ${i} with enough filler text to spend a few tokens</p>`
).join('')}</article></body></html>`
)
const capped = ax(['long-md.html', '--md'])
expect(capped.err).toContain('more result(s) hidden')

const all = ax(['long-md.html', '--md', '--all'])
expect(all.code).toBe(0)
expect(all.err).not.toContain('hidden')
expect(all.out.match(/paragraph number/g)).toHaveLength(300)

// an explicit --budget still wins, even together with --all
const budgeted = ax(['long-md.html', '--md', '--all', '--budget', '200'])
expect(budgeted.err).toContain('more result(s) hidden')
expect(budgeted.out.length).toBeLessThan(all.out.length)
})

test('cap: --md alone stays capped by the default token budget', () => {
// Long-prose page: few enough lines that --limit 50 never fires, so what
// truncates here can only be the ~2000-token default budget.
writeFileSync(
join(dir, 'prose-md.html'),
`<html><body><article>${Array.from(
{ length: 40 },
(_, i) => `<p>paragraph ${i}: ${'filler words to pad this paragraph out. '.repeat(10)}</p>`
).join('')}</article></body></html>`
)
const r = ax(['prose-md.html', '--md'])
expect(r.code).toBe(0)
expect(r.err).toContain('more result(s) hidden')
expect(Number(r.err.match(/continue with --offset (\d+)/)![1]!)).toBeLessThan(50)
expect(r.out.length).toBeLessThan(2000 * 4 + 500)
})

test('cap: --offset applies to JSON rows too', () => {
const r = ax(['page.html', '.card', '--row', 'title=a', '--json', '--offset', '1'])
expect(JSON.parse(r.out)).toEqual([{ title: 'Two' }])
Expand Down
Loading