From bd416acf2ed14fa4af7f4c82d835ac9c4f807af0 Mon Sep 17 00:00:00 2001 From: jonastmb <20173713+kopachlager@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:49:02 +0300 Subject: [PATCH] prepare CLI 0.1.3 release --- .../github-actions/crawler-readability.yml | 6 +- package-lock.json | 4 +- package.json | 2 +- src/cli.js | 10 +- src/fetch-public.js | 130 +++++++++++------- test/cli.test.js | 42 +++++- test/compare.test.js | 5 + test/fetch-public.test.js | 104 ++++++++++++++ test/release.test.js | 36 +++++ 9 files changed, 281 insertions(+), 58 deletions(-) create mode 100644 test/release.test.js diff --git a/examples/github-actions/crawler-readability.yml b/examples/github-actions/crawler-readability.yml index 1ed13f6..65cdc05 100644 --- a/examples/github-actions/crawler-readability.yml +++ b/examples/github-actions/crawler-readability.yml @@ -20,20 +20,20 @@ jobs: - name: Check crawler-readable HTML run: >- - npx --yes @prerenderbuddy/cli@0.1.2 + npx --yes @prerenderbuddy/cli@0.1.3 check "$SITE_URL" --user-agent googlebot --fail-on critical - name: Compare standard and AI crawler HTTP responses run: >- - npx --yes @prerenderbuddy/cli@0.1.2 + npx --yes @prerenderbuddy/cli@0.1.3 compare "$SITE_URL" --user-agent gptbot --fail-on critical - name: Validate discovery files run: >- - npx --yes @prerenderbuddy/cli@0.1.2 + npx --yes @prerenderbuddy/cli@0.1.3 files "$SITE_URL" --fail-on critical diff --git a/package-lock.json b/package-lock.json index bb58d4c..c6d3508 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@prerenderbuddy/cli", - "version": "0.1.2", + "version": "0.1.3", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@prerenderbuddy/cli", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "bin": { "prerenderbuddy": "bin/prerenderbuddy.js" diff --git a/package.json b/package.json index 316c5fe..38a4ea1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@prerenderbuddy/cli", - "version": "0.1.2", + "version": "0.1.3", "description": "Open-source crawler-readability and discovery-file diagnostics for public websites.", "homepage": "https://github.com/kopachlager/prerenderbuddy-cli#readme", "repository": { diff --git a/src/cli.js b/src/cli.js index 8f2518a..b002767 100644 --- a/src/cli.js +++ b/src/cli.js @@ -20,7 +20,8 @@ Options: --user-agent browser, googlebot, bingbot, gptbot, or claudebot --timeout request timeout from 1000 to 60000 (default: 15000) --text-ratio-threshold - compare text-volume tolerance from 0.01 to 0.99 (default: 0.30) + compare command only; text-volume tolerance from 0.01 to 0.99 + (default: 0.30) --json print machine-readable JSON --fail-on warning or critical --help show this help @@ -42,6 +43,7 @@ function parseArgs(args) { userAgent: 'googlebot', timeoutMs: 15_000, textRatioThreshold: 0.3, + textRatioThresholdProvided: false, json: false, failOn: null, }; @@ -56,6 +58,7 @@ function parseArgs(args) { else if (value === '--timeout') options.timeoutMs = Number(optionValue(args, index++, value)); else if (value === '--text-ratio-threshold') { options.textRatioThreshold = Number(optionValue(args, index++, value)); + options.textRatioThresholdProvided = true; } else if (value === '--fail-on') options.failOn = optionValue(args, index++, value); else if (value.startsWith('-')) throw new Error(`Unknown option "${value}".`); else positional.push(value); @@ -114,6 +117,9 @@ export async function runCliWithRuntime(args, runtime = {}) { } if (!url) throw new Error(`The ${command} command requires a URL.`); if (positional.length > 2) throw new Error('Only one URL can be checked at a time in v0.1.'); + if (options.textRatioThresholdProvided && command !== 'compare') { + throw new Error('--text-ratio-threshold is supported by the compare command only.'); + } const runOptions = { userAgent: options.userAgent, @@ -132,7 +138,7 @@ export function executionErrorResult(error) { ? 'timeout' : /private|blocked network|local and private|credentials|only http and https/i.test(message) ? 'unsafe_target' - : /unknown|requires|must be|only one URL|public URL is required|invalid url/i.test(message) + : /unknown|requires|must be|only one URL|compare command only|public URL is required|invalid url/i.test(message) ? 'invalid_input' : 'request_failed'; return { diff --git a/src/fetch-public.js b/src/fetch-public.js index f25a1c4..5acc6f0 100644 --- a/src/fetch-public.js +++ b/src/fetch-public.js @@ -7,9 +7,38 @@ export async function readBoundedText(response, maxChars = DEFAULT_MAX_CHARS) { return (await readBoundedResult(response, maxChars)).text; } -async function readBoundedResult(response, maxChars = DEFAULT_MAX_CHARS) { +function abortError() { + const error = new Error('The operation was aborted.'); + error.name = 'AbortError'; + return error; +} + +async function waitForAbort(promise, signal) { + if (!signal) return promise; + if (signal.aborted) throw abortError(); + + let onAbort; + const aborted = new Promise((_, reject) => { + onAbort = () => reject(abortError()); + signal.addEventListener('abort', onAbort, { once: true }); + }); + + try { + return await Promise.race([promise, aborted]); + } finally { + signal.removeEventListener('abort', onAbort); + } +} + +async function cancelBody(body, signal) { + if (!body) return; + const cancellation = body.cancel().catch(() => {}); + await waitForAbort(cancellation, signal); +} + +async function readBoundedResult(response, maxChars = DEFAULT_MAX_CHARS, signal) { if (!response.body) { - const text = await response.text(); + const text = await waitForAbort(response.text(), signal); return { text: text.slice(0, maxChars), truncated: text.length > maxChars }; } @@ -20,7 +49,7 @@ async function readBoundedResult(response, maxChars = DEFAULT_MAX_CHARS) { try { while (output.length <= maxChars) { - const { done, value } = await reader.read(); + const { done, value } = await waitForAbort(reader.read(), signal); if (done) { completed = true; break; @@ -30,7 +59,11 @@ async function readBoundedResult(response, maxChars = DEFAULT_MAX_CHARS) { output += decoder.decode(); return { text: output.slice(0, maxChars), truncated: output.length > maxChars }; } finally { - if (!completed) await reader.cancel().catch(() => {}); + if (!completed) { + const cancellation = reader.cancel().catch(() => {}); + if (signal?.aborted) void cancellation; + else await waitForAbort(cancellation, signal); + } reader.releaseLock(); } } @@ -44,62 +77,65 @@ export async function fetchPublicText(target, options = {}) { maxRedirects = 5, fetchFn = fetch, assertUrlFn = assertPublicUrl, + setTimeoutFn = setTimeout, + clearTimeoutFn = clearTimeout, } = options; let currentUrl = normalizePublicUrl(target); + const controller = new AbortController(); + const timeout = setTimeoutFn(() => controller.abort(), timeoutMs); - for (let redirects = 0; redirects <= maxRedirects; redirects += 1) { - await assertUrlFn(currentUrl); - const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), timeoutMs); - let response; - - try { - response = await fetchFn(currentUrl, { + try { + for (let redirects = 0; redirects <= maxRedirects; redirects += 1) { + await waitForAbort(Promise.resolve().then(() => assertUrlFn(currentUrl)), controller.signal); + const response = await waitForAbort(fetchFn(currentUrl, { redirect: 'manual', signal: controller.signal, headers: { Accept: accept, 'User-Agent': userAgent || 'PrerenderBuddyCLI/0.1 (+https://prerenderbuddy.com)', }, - }); - } catch (error) { - if (error?.name === 'AbortError') throw new Error(`Request timed out after ${timeoutMs} ms.`); - throw error; - } finally { - clearTimeout(timeout); - } + }), controller.signal); - if (!REDIRECT_CODES.has(response.status)) { - const body = await readBoundedResult(response, maxChars); - return { - requestedUrl: normalizePublicUrl(target), - finalUrl: currentUrl, - statusCode: response.status, - ok: response.ok, - contentType: response.headers.get('content-type') || '', - text: body.text, - truncated: body.truncated, - maxChars, - }; + if (!REDIRECT_CODES.has(response.status)) { + const body = await readBoundedResult(response, maxChars, controller.signal); + return { + requestedUrl: normalizePublicUrl(target), + finalUrl: currentUrl, + statusCode: response.status, + ok: response.ok, + contentType: response.headers.get('content-type') || '', + text: body.text, + truncated: body.truncated, + maxChars, + }; + } + + await cancelBody(response.body, controller.signal); + const location = response.headers.get('location'); + if (!location) { + return { + requestedUrl: normalizePublicUrl(target), + finalUrl: currentUrl, + statusCode: response.status, + ok: response.ok, + contentType: response.headers.get('content-type') || '', + text: '', + truncated: false, + maxChars, + }; + } + if (redirects === maxRedirects) throw new Error('Too many redirects while checking this URL.'); + currentUrl = normalizePublicUrl(new URL(location, currentUrl).toString()); } - const location = response.headers.get('location'); - if (!location) { - return { - requestedUrl: normalizePublicUrl(target), - finalUrl: currentUrl, - statusCode: response.status, - ok: response.ok, - contentType: response.headers.get('content-type') || '', - text: '', - truncated: false, - maxChars, - }; + throw new Error('Too many redirects while checking this URL.'); + } catch (error) { + if (controller.signal.aborted || error?.name === 'AbortError') { + throw new Error(`Request timed out after ${timeoutMs} ms.`); } - if (redirects === maxRedirects) throw new Error('Too many redirects while checking this URL.'); - currentUrl = normalizePublicUrl(new URL(location, currentUrl).toString()); + throw error; + } finally { + clearTimeoutFn(timeout); } - - throw new Error('Too many redirects while checking this URL.'); } diff --git a/test/cli.test.js b/test/cli.test.js index 55d0142..dd8db13 100644 --- a/test/cli.test.js +++ b/test/cli.test.js @@ -11,7 +11,7 @@ import { test('parses common CI options', () => { const result = parseArgs([ - 'check', + 'compare', 'https://example.com', '--user-agent', 'gptbot', @@ -24,18 +24,19 @@ test('parses common CI options', () => { '0.2', ]); - assert.deepEqual(result.positional, ['check', 'https://example.com']); + assert.deepEqual(result.positional, ['compare', 'https://example.com']); assert.equal(result.options.userAgent, 'gptbot'); assert.equal(result.options.timeoutMs, 5000); assert.equal(result.options.json, true); assert.equal(result.options.failOn, 'critical'); assert.equal(result.options.textRatioThreshold, 0.2); + assert.equal(result.options.textRatioThresholdProvided, true); }); test('rejects unsafe timeout and failure threshold values', () => { assert.throws(() => parseArgs(['check', 'example.com', '--timeout', '10']), /between 1000 and 60000/); assert.throws(() => parseArgs(['check', 'example.com', '--fail-on', 'pass']), /warning or critical/); - assert.throws(() => parseArgs(['check', 'example.com', '--text-ratio-threshold', '2']), /between 0.01 and 0.99/); + assert.throws(() => parseArgs(['compare', 'example.com', '--text-ratio-threshold', '2']), /between 0.01 and 0.99/); assert.throws(() => parseArgs(['check', 'example.com', '--user-agent']), /requires a value/); assert.throws(() => parseArgs(['check', 'example.com', '--user-agent', 'unknown']), /Unknown user-agent/); assert.throws(() => parseArgs(['check', 'example.com', '--unknown']), /Unknown option/); @@ -102,6 +103,15 @@ test('passes compare thresholds to the command handler', async () => { assert.equal(received.textRatioThreshold, 0.15); }); +test('rejects the compare-only text ratio option for check and files', async () => { + for (const command of ['check', 'files']) { + await assert.rejects( + () => runCli([command, 'https://example.com', '--text-ratio-threshold', '0.2']), + /compare command only/, + ); + } +}); + test('rejects invalid commands, missing URLs, and extra positional arguments', async () => { await assert.rejects(() => runCli(['unknown', 'https://example.com']), /Unknown command/); await assert.rejects(() => runCli(['check']), /requires a URL/); @@ -141,3 +151,29 @@ test('the executable keeps human execution errors on stderr', () => { assert.equal(result.stdout, ''); assert.match(result.stderr, /Prerender Buddy check failed: Unknown command/); }); + +test('unsupported compare options produce JSON and human execution errors', () => { + const binary = fileURLToPath(new URL('../bin/prerenderbuddy.js', import.meta.url)); + const json = spawnSync(process.execPath, [ + binary, + 'check', + 'https://example.com', + '--text-ratio-threshold', + '0.2', + '--json', + ], { encoding: 'utf8' }); + assert.equal(json.status, 2); + assert.equal(json.stderr, ''); + assert.equal(JSON.parse(json.stdout).error.code, 'invalid_input'); + + const human = spawnSync(process.execPath, [ + binary, + 'files', + 'https://example.com', + '--text-ratio-threshold', + '0.2', + ], { encoding: 'utf8' }); + assert.equal(human.status, 2); + assert.equal(human.stdout, ''); + assert.match(human.stderr, /compare command only/); +}); diff --git a/test/compare.test.js b/test/compare.test.js index 72d14b7..093fbb0 100644 --- a/test/compare.test.js +++ b/test/compare.test.js @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import { readFile } from 'node:fs/promises'; import test from 'node:test'; import { compareUrl, contentDelta } from '../src/compare.js'; +import { formatHuman } from '../src/format.js'; import { analyzeHtml } from '../src/html.js'; async function fixture(name) { @@ -51,6 +52,10 @@ test('reports exact metadata and text-volume differences separately', async () = assert.ok(result.issues.some((issue) => ( issue.code === 'crawler_response_differs' && issue.compatibilityAlias ))); + assert.match(JSON.stringify(result), /crawler_response_differs/); + const human = formatHuman(result); + assert.match(human, /text_volume_differs/); + assert.doesNotMatch(human, /crawler_response_differs/); }); test('status differences and crawler app shells remain critical', async () => { diff --git a/test/fetch-public.test.js b/test/fetch-public.test.js index bf89714..bfa8419 100644 --- a/test/fetch-public.test.js +++ b/test/fetch-public.test.js @@ -57,6 +57,60 @@ test('reports timeouts separately from response diagnostics', async () => { ); }); +test('times out when fetch never returns even if the fetch implementation ignores the signal', async () => { + let timerCleared = false; + await assert.rejects( + () => fetchPublicText('https://example.com/never', { + timeoutMs: 5, + assertUrlFn: async () => {}, + fetchFn: async () => new Promise(() => {}), + setTimeoutFn: (callback, delay) => setTimeout(callback, delay), + clearTimeoutFn: (handle) => { + timerCleared = true; + clearTimeout(handle); + }, + }), + /Request timed out after 5 ms\./, + ); + assert.equal(timerCleared, true); +}); + +test('includes public URL validation in the lifecycle timeout', async () => { + await assert.rejects( + () => fetchPublicText('https://example.com/dns-stall', { + timeoutMs: 5, + assertUrlFn: async () => new Promise(() => {}), + fetchFn: async () => { + throw new Error('fetch should not run'); + }, + }), + /Request timed out after 5 ms\./, + ); +}); + +test('keeps the timeout active while a response body is stalled', async () => { + let cancelled = false; + const body = new ReadableStream({ + pull: () => new Promise(() => {}), + cancel: () => { + cancelled = true; + }, + }); + + await assert.rejects( + () => fetchPublicText('https://example.com/stalled-body', { + timeoutMs: 5, + assertUrlFn: async () => {}, + fetchFn: async () => new Response(body, { + status: 200, + headers: { 'content-type': 'text/plain' }, + }), + }), + /Request timed out after 5 ms\./, + ); + assert.equal(cancelled, true); +}); + test('preserves response content type and non-truncated state', async () => { const result = await fetchPublicText('https://example.com/data', { assertUrlFn: async () => {}, @@ -69,6 +123,56 @@ test('preserves response content type and non-truncated state', async () => { assert.equal(result.truncated, false); }); +test('reads a normal streamed response and clears its lifecycle timer', async () => { + const encoder = new TextEncoder(); + const body = new ReadableStream({ + start(controller) { + controller.enqueue(encoder.encode('streamed ')); + controller.enqueue(encoder.encode('response')); + controller.close(); + }, + }); + const timerHandle = Symbol('timer'); + const cleared = []; + + const result = await fetchPublicText('https://example.com/stream', { + assertUrlFn: async () => {}, + fetchFn: async () => new Response(body, { + status: 200, + headers: { 'content-type': 'text/plain' }, + }), + setTimeoutFn: () => timerHandle, + clearTimeoutFn: (handle) => cleared.push(handle), + }); + + assert.equal(result.text, 'streamed response'); + assert.equal(result.truncated, false); + assert.deepEqual(cleared, [timerHandle]); +}); + +test('cancels a streamed response when the size limit is reached', async () => { + const encoder = new TextEncoder(); + let cancelled = false; + const body = new ReadableStream({ + pull(controller) { + controller.enqueue(encoder.encode('1234567890')); + }, + cancel() { + cancelled = true; + }, + }); + + const result = await fetchPublicText('https://example.com/large', { + maxChars: 5, + assertUrlFn: async () => {}, + fetchFn: async () => new Response(body, { status: 200 }), + }); + + assert.equal(result.text, '12345'); + assert.equal(result.truncated, true); + assert.equal(cancelled, true); +}); + test('handles responses without a stream and redirects without a location', async () => { const noBodyResponse = { body: null, diff --git a/test/release.test.js b/test/release.test.js new file mode 100644 index 0000000..a5795f6 --- /dev/null +++ b/test/release.test.js @@ -0,0 +1,36 @@ +import assert from 'node:assert/strict'; +import { readFile } from 'node:fs/promises'; +import test from 'node:test'; + +async function text(path) { + return readFile(new URL(`../${path}`, import.meta.url), 'utf8'); +} + +test('version and public workflow examples are ready for v0.1.3', async () => { + const packageJson = JSON.parse(await text('package.json')); + const packageLock = JSON.parse(await text('package-lock.json')); + const example = await text('examples/github-actions/crawler-readability.yml'); + + assert.equal(packageJson.version, '0.1.3'); + assert.equal(packageLock.version, '0.1.3'); + assert.equal(packageLock.packages[''].version, '0.1.3'); + assert.equal((example.match(/@prerenderbuddy\/cli@0\.1\.3/g) || []).length, 3); + assert.doesNotMatch(example, /@prerenderbuddy\/cli@0\.1\.2/); +}); + +test('trusted release publishing remains repository-only and protected', async () => { + const workflow = await text('.github/workflows/publish.yml'); + + assert.match(workflow, /release:\s*\n\s+types: \[published\]/); + assert.doesNotMatch(workflow, /pull_request:/); + assert.match(workflow, /github\.repository == 'kopachlager\/prerenderbuddy-cli'/); + assert.match(workflow, /contents: read/); + assert.match(workflow, /id-token: write/); + assert.match(workflow, /npm ci --ignore-scripts/); + assert.match(workflow, /npm test/); + assert.match(workflow, /npm run check/); + assert.match(workflow, /npm run pack:check/); + assert.match(workflow, /GITHUB_REF_NAME/); + assert.match(workflow, /npm publish --provenance --access public/); + assert.doesNotMatch(workflow, /NPM_TOKEN|NODE_AUTH_TOKEN/); +});