diff --git a/.github/release.yml b/.github/release.yml
new file mode 100644
index 0000000..a493214
--- /dev/null
+++ b/.github/release.yml
@@ -0,0 +1,15 @@
+changelog:
+ categories:
+ - title: Diagnostic improvements
+ labels:
+ - enhancement
+ - title: Fixes
+ labels:
+ - bug
+ - title: Documentation and maintenance
+ labels:
+ - documentation
+ - dependencies
+ - title: Other changes
+ labels:
+ - "*"
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c40753e..3ef2a1d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,12 +15,13 @@ jobs:
node-version: [20, 22, 24]
runs-on: ubuntu-latest
steps:
- - uses: actions/checkout@v7
- - uses: actions/setup-node@v7
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: ${{ matrix.node-version }}
cache: npm
- run: npm ci --ignore-scripts
- run: npm test
+ - run: npm run test:coverage
- run: npm run check
- run: npm run pack:check
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 66593d6..bc1f146 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -20,9 +20,9 @@ jobs:
contents: read
id-token: write
steps:
- - uses: actions/checkout@v7
+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- - uses: actions/setup-node@v7
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
registry-url: "https://registry.npmjs.org"
@@ -37,6 +37,9 @@ jobs:
- name: Check source syntax
run: npm run check
+ - name: Verify package contents
+ run: npm run pack:check
+
- name: Verify release tag matches package version
shell: bash
run: |
@@ -48,4 +51,4 @@ jobs:
fi
- name: Publish to npm with trusted publishing
- run: npm publish --access public
+ run: npm publish --provenance --access public
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 22ef47b..1aad0ba 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -26,6 +26,7 @@ Use Node.js 20 or newer.
```bash
npm test
+npm run test:coverage
npm run check
npm run pack:check
```
diff --git a/README.md b/README.md
index ec79742..b3b4bb5 100644
--- a/README.md
+++ b/README.md
@@ -8,31 +8,60 @@
Open-source diagnostics for checking what public websites return to crawlers.
-The CLI inspects returned HTML, compares browser-style and crawler-style responses, and validates common discovery files. It does not render JavaScript, change a website, require a Prerender Buddy account, or predict search rankings, indexing, AI citations, mentions, or traffic.
+The CLI inspects returned HTML, compares standard and crawler user-agent HTTP responses, and validates common discovery files. It does not render JavaScript, change a website, require a Prerender Buddy account, or predict search rankings, indexing, AI citations, mentions, or traffic.
-This is an early public release. Install it from npm or run it from a local checkout.
+Run it without installing:
+
+```bash
+npx @prerenderbuddy/cli check https://example.com --user-agent googlebot
+```
+
+Or install the command globally:
+
+```bash
+npm install --global @prerenderbuddy/cli
+prerenderbuddy check https://example.com
+```
+
+Example output from the included loading-placeholder fixture:
+
+```text
+Prerender Buddy · crawler HTML check · CRITICAL
+URL https://example.com/app
+Crawler profile Googlebot
+HTTP 200
+Final URL https://example.com/app
+Title Loading application
+Description Application loading screen.
+H1 Loading application
+Readable text 41 characters / 5 words
+App-shell signs root div, bundled assets, module scripts
+
+Issues:
+- CRITICAL [app_shell]: Returned HTML has limited visible content and multiple JavaScript app-shell signals.
+ Why: Crawlers that do not execute JavaScript may receive only the application shell.
+ Evidence: {"readableCharacters":41,"scriptCount":1,"signals":["loading-only visible text","module or bundled application script","root div detected","bundled assets detected","module scripts detected"]}
+ Next: Inspect the raw response and test whether important page content is present before JavaScript executes.
+
+This checks returned HTML only. It does not predict rankings, indexing, citations, mentions, or traffic.
+```
+
+Reproduce that output from a local checkout with `npm run demo:fixture`. The demo injects a static fixture into the normal check and formatting functions; it does not weaken public-URL safety or start a local URL-fetching service.
+
+The finding is a documented heuristic, not proof that a crawler failed. The CLI does not run Chromium, execute page JavaScript, or produce rendered HTML.
## Requirements
- Node.js 20 or newer
- A public HTTP or HTTPS URL
-Run without installing:
+Other commands:
```bash
-npx @prerenderbuddy/cli check https://example.com
npx @prerenderbuddy/cli compare https://example.com --user-agent gptbot
npx @prerenderbuddy/cli files https://example.com
```
-Local checkout usage:
-
-```bash
-node ./bin/prerenderbuddy.js check https://example.com
-node ./bin/prerenderbuddy.js compare https://example.com --user-agent gptbot
-node ./bin/prerenderbuddy.js files https://example.com
-```
-
## Commands
### Check crawler-readable HTML
@@ -57,9 +86,17 @@ The app-shell test is a heuristic. A warning is a reason to inspect the page, no
prerenderbuddy compare https://example.com --user-agent gptbot
```
-Compares a browser-style response with the selected crawler response. It flags status, metadata, heading, and material text differences. Different output can be legitimate; the result is evidence to review, not an accusation of cloaking.
+Compares a browser-style user-agent HTTP response with the selected crawler user-agent HTTP response. It reports status, metadata, heading, and material text-volume differences separately. Different output can be legitimate; the result is evidence to review, not an accusation of cloaking.
+
+Both sides are ordinary HTTP responses. Neither side executes JavaScript. This is not a raw-versus-browser-rendered comparison, and the package has no browser engine or connection to Prerender Buddy’s private rendering infrastructure.
-This is not a raw-versus-browser-rendered comparison. The open-source v0.1 package deliberately has no browser engine or connection to Prerender Buddy’s private rendering infrastructure.
+The default text-ratio tolerance is 30% in either direction. Adjust it for a known-variable site:
+
+```bash
+prerenderbuddy compare https://example.com --text-ratio-threshold 0.20
+```
+
+The comparison normalizes HTML into whitespace-collapsed visible text and reports the exact lengths and metadata values that changed. It does not perform semantic AI comparison or automatically remove cookie notices, timestamps, rotating banners, experiments, personalization, regional content, anti-bot pages, or temporary CDN responses. Review those sources of variation before treating a warning as a regression.
### Validate discovery files
@@ -101,6 +138,29 @@ Exit codes:
JSON fields are intended to become stable at `1.0.0`. Before then, minor releases may add or refine diagnostic fields.
+### Programmatic use
+
+The same diagnostics are exported as dependency-free ESM functions:
+
+```js
+import {
+ analyzeHtml,
+ checkDiscoveryFiles,
+ checkUrl,
+ compareUrl,
+} from '@prerenderbuddy/cli';
+
+const page = await checkUrl('https://example.com', { userAgent: 'googlebot' });
+const comparison = await compareUrl('https://example.com', {
+ userAgent: 'gptbot',
+ textRatioThreshold: 0.2,
+});
+const files = await checkDiscoveryFiles('https://example.com');
+const localAnalysis = analyzeHtml('Example
');
+```
+
+Network functions retain the same public-URL safety, redirect, timeout, and response-size controls as the CLI.
+
### GitHub Actions
Copy [`examples/github-actions/crawler-readability.yml`](https://github.com/kopachlager/prerenderbuddy-cli/blob/main/examples/github-actions/crawler-readability.yml)
@@ -110,7 +170,7 @@ into the target repository as `.github/workflows/crawler-readability.yml`, then
The example:
- checks crawler-readable HTML as Googlebot;
-- compares browser-style and GPTBot responses;
+- compares browser-style user-agent and GPTBot HTTP responses;
- validates `robots.txt`, `sitemap.xml`, and `llms.txt`;
- fails only on critical findings by default;
- pins the CLI version so updates are reviewed deliberately.
@@ -131,16 +191,39 @@ Fetched page text is untrusted data. The CLI displays and analyses it; it must n
See [SECURITY.md](./SECURITY.md) for reporting and current limitations.
-## When a managed service is not needed
+## CLI and hosted service
+
+| Capability | Open-source CLI | Hosted Prerender Buddy |
+| --- | --- | --- |
+| One-time public URL diagnostics | Yes | Yes |
+| Local execution and CI | Yes | No |
+| Returned HTML inspection | Yes | Yes |
+| JavaScript execution | No | Yes, for managed crawler-ready rendering |
+| Scheduled monitoring | No | Yes |
+| Baselines, history, and incidents | No | Yes |
+| Managed crawler routing | No | Yes |
+| Cache operations | No | Yes |
+| DNS or proxy onboarding | No | Yes |
+| Account required | No | Yes |
+
+The CLI is independently useful for diagnostics. The hosted service operates rendering, routing, monitoring, and cache workflows when testing shows that a production deployment needs them.
+
+### When a managed service is not needed
If important production routes already return complete, consistent HTML to the crawlers you care about, an additional rendering layer may not be needed. Continue testing after framework, hosting, domain, or deployment changes.
-## When Prerender Buddy may help
+### When Prerender Buddy may help
If production tests find missing, partial, crawler-dependent, or unreliable HTML, Prerender Buddy can provide managed crawler-ready rendering. Its hosted service also provides scheduled monitoring, baselines, incidents, history, cache operations, DNS/proxy onboarding, crawler routing, and support.
The CLI diagnoses a current response. The hosted service operates and monitors the production solution.
+## Fixtures and heuristic limits
+
+Deterministic fixtures live in [`test/fixtures`](./test/fixtures). They cover healthy HTML, thin application shells, minimal static pages, canvas applications, loading placeholders, hidden script data, cookie banners, crawler-blocked responses, malformed metadata, and discovery-file errors.
+
+Application-shell detection uses observable inputs: readable character count, empty `root` or `app` mount points, loading-only text, module or bundled scripts, and framework markers. It does not identify a framework failure, simulate verified crawler traffic, or prove that a genuine crawler received the same response.
+
## Development
```bash
@@ -156,6 +239,14 @@ The package intentionally starts with no runtime dependencies.
Read [CONTRIBUTING.md](./CONTRIBUTING.md). Keep contributions focused on accurate, reproducible crawler diagnostics. New checks need fixtures, tests, documented limitations, and evidence that they do not duplicate managed-service operations.
+## Next steps
+
+- Run the [browser-based crawler checker](https://prerenderbuddy.com/tools/bot-view-checker).
+- Read the [technical documentation](https://prerenderbuddy.com/docs).
+- Review the [public roadmap](./ROADMAP.md).
+- Report reproducible CLI problems in [GitHub Issues](https://github.com/kopachlager/prerenderbuddy-cli/issues).
+- Use the [hosted Prerender Buddy service](https://prerenderbuddy.com) when diagnostics show that managed rendering or monitoring is needed.
+
## License
Apache License 2.0. See [LICENSE](./LICENSE).
diff --git a/ROADMAP.md b/ROADMAP.md
new file mode 100644
index 0000000..669026f
--- /dev/null
+++ b/ROADMAP.md
@@ -0,0 +1,24 @@
+# Public roadmap
+
+This roadmap describes possible directions, not promised dates or release commitments.
+
+## Diagnostic improvements
+
+- additional transparent crawler profiles supported by public documentation;
+- clearer evidence and remediation fields for every finding;
+- more deterministic HTML and discovery-file fixtures;
+- configurable diagnostic thresholds;
+- stable JSON schema work toward `1.0`;
+- URL-list or sitemap-driven batch checks;
+- richer CI annotations and possible SARIF output.
+
+## Explicitly out of scope
+
+- browser rendering or JavaScript execution;
+- private Prerender Buddy API access;
+- hosted monitoring, baselines, incidents, or history;
+- managed crawler routing or DNS onboarding;
+- cache management;
+- proxy, queue, billing, or infrastructure deployment.
+
+The CLI will remain usable without an account, authentication, telemetry, or calls to Prerender Buddy production services.
diff --git a/bin/prerenderbuddy.js b/bin/prerenderbuddy.js
index f1a0b32..244c0b2 100755
--- a/bin/prerenderbuddy.js
+++ b/bin/prerenderbuddy.js
@@ -1,8 +1,12 @@
#!/usr/bin/env node
-import { runCli } from '../src/cli.js';
+import { executionErrorResult, runCli } from '../src/cli.js';
runCli(process.argv.slice(2)).catch((error) => {
- process.stderr.write(`Prerender Buddy check failed: ${error.message}\n`);
+ if (process.argv.includes('--json')) {
+ process.stdout.write(`${JSON.stringify(executionErrorResult(error), null, 2)}\n`);
+ } else {
+ process.stderr.write(`Prerender Buddy check failed: ${error.message}\n`);
+ }
process.exitCode = 2;
});
diff --git a/examples/github-actions/crawler-readability.yml b/examples/github-actions/crawler-readability.yml
index 5be83af..1ed13f6 100644
--- a/examples/github-actions/crawler-readability.yml
+++ b/examples/github-actions/crawler-readability.yml
@@ -14,26 +14,26 @@ jobs:
env:
SITE_URL: https://example.com
steps:
- - uses: actions/setup-node@v7
+ - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "24"
- name: Check crawler-readable HTML
run: >-
- npx --yes @prerenderbuddy/cli@0.1.1
+ npx --yes @prerenderbuddy/cli@0.1.2
check "$SITE_URL"
--user-agent googlebot
--fail-on critical
- - name: Compare browser and AI crawler responses
+ - name: Compare standard and AI crawler HTTP responses
run: >-
- npx --yes @prerenderbuddy/cli@0.1.1
+ npx --yes @prerenderbuddy/cli@0.1.2
compare "$SITE_URL"
--user-agent gptbot
--fail-on critical
- name: Validate discovery files
run: >-
- npx --yes @prerenderbuddy/cli@0.1.1
+ npx --yes @prerenderbuddy/cli@0.1.2
files "$SITE_URL"
--fail-on critical
diff --git a/package.json b/package.json
index 164a362..316c5fe 100644
--- a/package.json
+++ b/package.json
@@ -23,25 +23,30 @@
"LICENSE",
"NOTICE",
"README.md",
+ "ROADMAP.md",
"SECURITY.md"
],
"scripts": {
"check": "node --check bin/prerenderbuddy.js && node --check src/*.js",
+ "demo:fixture": "node scripts/demo-fixture.js",
"test": "node --test",
"test:coverage": "node --test --experimental-test-coverage",
- "pack:check": "npm pack --dry-run"
+ "pack:check": "node scripts/verify-package.js"
},
"engines": {
"node": ">=20"
},
"keywords": [
"crawler",
+ "technical-seo",
"javascript-seo",
"prerendering",
- "robots.txt",
+ "googlebot",
+ "gptbot",
+ "robots-txt",
"sitemap",
- "llms.txt",
- "cli"
+ "llms-txt",
+ "seo-cli"
],
"license": "Apache-2.0",
"publishConfig": {
diff --git a/scripts/demo-fixture.js b/scripts/demo-fixture.js
new file mode 100644
index 0000000..69490d5
--- /dev/null
+++ b/scripts/demo-fixture.js
@@ -0,0 +1,18 @@
+import { readFile } from 'node:fs/promises';
+import { checkUrl } from '../src/check.js';
+import { formatHuman } from '../src/format.js';
+
+const html = await readFile(
+ new URL('../test/fixtures/html/loading-placeholder.html', import.meta.url),
+ 'utf8',
+);
+const result = await checkUrl('https://example.com/app', {
+ userAgent: 'googlebot',
+ assertUrlFn: async () => {},
+ fetchFn: async () => new Response(html, {
+ status: 200,
+ headers: { 'content-type': 'text/html; charset=utf-8' },
+ }),
+});
+
+process.stdout.write(`${formatHuman(result)}\n`);
diff --git a/scripts/verify-package.js b/scripts/verify-package.js
new file mode 100644
index 0000000..8970ea0
--- /dev/null
+++ b/scripts/verify-package.js
@@ -0,0 +1,42 @@
+import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
+import { readFile } from 'node:fs/promises';
+
+const packageJson = JSON.parse(await readFile(new URL('../package.json', import.meta.url), 'utf8'));
+const packageLock = JSON.parse(await readFile(new URL('../package-lock.json', import.meta.url), 'utf8'));
+assert.equal(packageLock.name, packageJson.name, 'package-lock.json name must match package.json');
+assert.equal(packageLock.version, packageJson.version, 'package-lock.json version must match package.json');
+assert.equal(packageLock.packages[''].version, packageJson.version, 'lockfile root version must match package.json');
+
+const result = spawnSync('npm', ['pack', '--dry-run', '--json'], {
+ cwd: new URL('..', import.meta.url),
+ encoding: 'utf8',
+});
+
+if (result.status !== 0) {
+ process.stderr.write(result.stderr || result.stdout);
+ process.exit(result.status || 1);
+}
+
+const [manifest] = JSON.parse(result.stdout);
+const files = manifest.files.map((entry) => entry.path).sort();
+const required = [
+ 'LICENSE',
+ 'NOTICE',
+ 'README.md',
+ 'ROADMAP.md',
+ 'SECURITY.md',
+ 'bin/prerenderbuddy.js',
+ 'package.json',
+ 'src/index.js',
+];
+
+for (const file of required) assert.ok(files.includes(file), `Package is missing ${file}`);
+for (const file of files) {
+ assert.ok(
+ required.includes(file) || file.startsWith('src/'),
+ `Unexpected file in npm package: ${file}`,
+ );
+}
+
+process.stdout.write(`Verified ${files.length} package files for ${manifest.name}@${manifest.version}.\n`);
diff --git a/src/check.js b/src/check.js
index 833443e..96e97bc 100644
--- a/src/check.js
+++ b/src/check.js
@@ -9,6 +9,9 @@ export async function checkUrl(input, options = {}) {
const response = await fetchPublicText(url, {
userAgent: profile.value,
timeoutMs: options.timeoutMs,
+ fetchFn: options.fetchFn,
+ assertUrlFn: options.assertUrlFn,
+ maxChars: options.maxChars,
});
const html = analyzeHtml(response.text);
const issues = buildHtmlIssues(html, response);
@@ -23,6 +26,8 @@ export async function checkUrl(input, options = {}) {
ok: response.ok,
finalUrl: response.finalUrl,
contentType: response.contentType,
+ truncated: response.truncated,
+ maxChars: response.maxChars,
},
html,
issues,
diff --git a/src/cli.js b/src/cli.js
index 3fa4892..8f2518a 100644
--- a/src/cli.js
+++ b/src/cli.js
@@ -19,6 +19,8 @@ Usage:
Options:
--user-agent browser, googlebot, bingbot, gptbot, or claudebot
--timeout request timeout from 1000 to 60000 (default: 15000)
+ --text-ratio-threshold
+ compare text-volume tolerance from 0.01 to 0.99 (default: 0.30)
--json print machine-readable JSON
--fail-on warning or critical
--help show this help
@@ -27,8 +29,22 @@ Options:
This tool checks public responses. It does not predict rankings, indexing, citations,
mentions, or traffic, and it does not use Prerender Buddy's managed rendering service.`;
+function optionValue(args, index, option) {
+ const value = args[index + 1];
+ if (value === undefined || value.startsWith('--')) {
+ throw new Error(`${option} requires a value.`);
+ }
+ return value;
+}
+
function parseArgs(args) {
- const options = { userAgent: 'googlebot', timeoutMs: 15_000, json: false, failOn: null };
+ const options = {
+ userAgent: 'googlebot',
+ timeoutMs: 15_000,
+ textRatioThreshold: 0.3,
+ json: false,
+ failOn: null,
+ };
const positional = [];
for (let index = 0; index < args.length; index += 1) {
@@ -36,9 +52,11 @@ function parseArgs(args) {
if (value === '--json') options.json = true;
else if (value === '--help' || value === '-h') options.help = true;
else if (value === '--version' || value === '-v') options.version = true;
- else if (value === '--user-agent') options.userAgent = args[++index];
- else if (value === '--timeout') options.timeoutMs = Number(args[++index]);
- else if (value === '--fail-on') options.failOn = args[++index];
+ else if (value === '--user-agent') options.userAgent = optionValue(args, index++, value);
+ else if (value === '--timeout') options.timeoutMs = Number(optionValue(args, index++, value));
+ else if (value === '--text-ratio-threshold') {
+ options.textRatioThreshold = Number(optionValue(args, index++, value));
+ } else if (value === '--fail-on') options.failOn = optionValue(args, index++, value);
else if (value.startsWith('-')) throw new Error(`Unknown option "${value}".`);
else positional.push(value);
}
@@ -46,6 +64,11 @@ function parseArgs(args) {
if (!Number.isInteger(options.timeoutMs) || options.timeoutMs < 1_000 || options.timeoutMs > 60_000) {
throw new Error('--timeout must be an integer between 1000 and 60000.');
}
+ if (!Number.isFinite(options.textRatioThreshold)
+ || options.textRatioThreshold < 0.01
+ || options.textRatioThreshold > 0.99) {
+ throw new Error('--text-ratio-threshold must be a number between 0.01 and 0.99.');
+ }
if (options.failOn && !['warning', 'critical'].includes(options.failOn)) {
throw new Error('--fail-on must be warning or critical.');
}
@@ -62,14 +85,26 @@ function shouldFail(summary, threshold) {
return summary === 'critical';
}
-export async function runCli(args) {
+export async function runCli(args, runtime = {}) {
+ return runCliWithRuntime(args, runtime);
+}
+
+export async function runCliWithRuntime(args, runtime = {}) {
const { positional, options } = parseArgs(args);
+ const write = runtime.write || ((value) => process.stdout.write(value));
+ const setExitCode = runtime.setExitCode || ((value) => { process.exitCode = value; });
+ const handlers = {
+ check: checkUrl,
+ compare: compareUrl,
+ files: checkDiscoveryFiles,
+ ...runtime.handlers,
+ };
if (options.help || (!positional.length && !options.version)) {
- process.stdout.write(`${HELP}\n`);
+ write(`${HELP}\n`);
return;
}
if (options.version) {
- process.stdout.write(`${VERSION}\n`);
+ write(`${VERSION}\n`);
return;
}
@@ -80,15 +115,31 @@ export async function runCli(args) {
if (!url) throw new Error(`The ${command} command requires a URL.`);
if (positional.length > 2) throw new Error('Only one URL can be checked at a time in v0.1.');
- const runOptions = { userAgent: options.userAgent, timeoutMs: options.timeoutMs };
- const result = command === 'check'
- ? await checkUrl(url, runOptions)
- : command === 'compare'
- ? await compareUrl(url, runOptions)
- : await checkDiscoveryFiles(url, runOptions);
+ const runOptions = {
+ userAgent: options.userAgent,
+ timeoutMs: options.timeoutMs,
+ textRatioThreshold: options.textRatioThreshold,
+ };
+ const result = await handlers[command](url, runOptions);
+
+ write(`${options.json ? JSON.stringify(result, null, 2) : formatHuman(result)}\n`);
+ if (shouldFail(result.summary, options.failOn)) setExitCode(1);
+}
- process.stdout.write(`${options.json ? JSON.stringify(result, null, 2) : formatHuman(result)}\n`);
- if (shouldFail(result.summary, options.failOn)) process.exitCode = 1;
+export function executionErrorResult(error) {
+ const message = error instanceof Error ? error.message : String(error);
+ const code = /timed out/i.test(message)
+ ? 'timeout'
+ : /private|blocked network|local and private|credentials|only http and https/i.test(message)
+ ? 'unsafe_target'
+ : /unknown|requires|must be|only one URL|public URL is required|invalid url/i.test(message)
+ ? 'invalid_input'
+ : 'request_failed';
+ return {
+ command: null,
+ summary: 'error',
+ error: { code, message },
+ };
}
export { HELP, parseArgs, shouldFail };
diff --git a/src/compare.js b/src/compare.js
index 6649d6d..fb7b8f5 100644
--- a/src/compare.js
+++ b/src/compare.js
@@ -3,14 +3,32 @@ import { analyzeHtml } from './html.js';
import { getUserAgentProfile } from './profiles.js';
import { normalizePublicUrl } from './url-safety.js';
-function contentDelta(browser, crawler) {
- const baseline = Math.max(browser.textLength, 1);
+export function contentDelta(standard, crawler, textRatioThreshold = 0.3) {
+ const baseline = Math.max(standard.textLength, 1);
+ const minimum = Number((1 - textRatioThreshold).toFixed(2));
+ const maximum = Number((1 + textRatioThreshold).toFixed(2));
return {
- textLength: crawler.textLength - browser.textLength,
+ textLength: crawler.textLength - standard.textLength,
textRatio: Number((crawler.textLength / baseline).toFixed(2)),
- titleChanged: crawler.title !== browser.title,
- descriptionChanged: crawler.description !== browser.description,
- h1Changed: JSON.stringify(crawler.headings.h1) !== JSON.stringify(browser.headings.h1),
+ textRatioThreshold,
+ acceptedTextRatio: { minimum, maximum },
+ titleChanged: crawler.title !== standard.title,
+ descriptionChanged: crawler.description !== standard.description,
+ h1Changed: JSON.stringify(crawler.headings.h1) !== JSON.stringify(standard.headings.h1),
+ values: {
+ standard: {
+ textLength: standard.textLength,
+ title: standard.title,
+ description: standard.description,
+ h1: standard.headings.h1,
+ },
+ crawler: {
+ textLength: crawler.textLength,
+ title: crawler.title,
+ description: crawler.description,
+ h1: crawler.headings.h1,
+ },
+ },
};
}
@@ -18,18 +36,27 @@ export async function compareUrl(input, options = {}) {
const url = normalizePublicUrl(input);
const browserProfile = getUserAgentProfile('browser');
const crawlerProfile = getUserAgentProfile(options.userAgent);
+ const textRatioThreshold = options.textRatioThreshold ?? 0.3;
+ const fetchOptions = {
+ timeoutMs: options.timeoutMs,
+ fetchFn: options.fetchFn,
+ assertUrlFn: options.assertUrlFn,
+ maxChars: options.maxChars,
+ };
const [browserResponse, crawlerResponse] = await Promise.all([
- fetchPublicText(url, { userAgent: browserProfile.value, timeoutMs: options.timeoutMs }),
- fetchPublicText(url, { userAgent: crawlerProfile.value, timeoutMs: options.timeoutMs }),
+ fetchPublicText(url, { ...fetchOptions, userAgent: browserProfile.value }),
+ fetchPublicText(url, { ...fetchOptions, userAgent: crawlerProfile.value }),
]);
const browser = analyzeHtml(browserResponse.text);
const crawler = analyzeHtml(crawlerResponse.text);
- const difference = contentDelta(browser, crawler);
+ const difference = contentDelta(browser, crawler, textRatioThreshold);
+ const textVolumeDiffers = difference.textRatio < difference.acceptedTextRatio.minimum
+ || difference.textRatio > difference.acceptedTextRatio.maximum;
const materiallyDifferent = browserResponse.statusCode !== crawlerResponse.statusCode
- || difference.textRatio < 0.7
- || difference.textRatio > 1.3
+ || textVolumeDiffers
|| difference.titleChanged
+ || difference.descriptionChanged
|| difference.h1Changed;
const issues = [];
@@ -37,21 +64,74 @@ export async function compareUrl(input, options = {}) {
issues.push({
severity: 'critical',
code: 'status_differs',
- message: `Browser and ${crawlerProfile.label} responses return different status codes.`,
+ message: `Standard and ${crawlerProfile.label} HTTP responses return different status codes.`,
+ why: 'Different status codes can change whether the page is accessible to the selected crawler.',
+ evidence: {
+ standardStatusCode: browserResponse.statusCode,
+ crawlerStatusCode: crawlerResponse.statusCode,
+ },
+ nextStep: 'Confirm whether crawler-specific status handling is intentional and stable.',
});
}
- if (materiallyDifferent) {
+ if (textVolumeDiffers) {
issues.push({
severity: 'warning',
- code: 'crawler_response_differs',
- message: `The ${crawlerProfile.label} response differs materially from the browser-style response; review whether the difference is intended.`,
+ code: 'text_volume_differs',
+ message: `The ${crawlerProfile.label} response has a materially different readable-text volume.`,
+ why: 'A large text-volume difference can indicate missing content, an interstitial, personalization, or intentional crawler handling.',
+ evidence: {
+ standardCharacters: browser.textLength,
+ crawlerCharacters: crawler.textLength,
+ textRatio: difference.textRatio,
+ acceptedTextRatio: difference.acceptedTextRatio,
+ },
+ nextStep: 'Compare the returned text and rule out banners, regional content, experiments, authentication, or temporary edge responses.',
+ });
+ }
+ for (const [changed, code, label, standardValue, crawlerValue] of [
+ [difference.titleChanged, 'title_differs', 'title', browser.title, crawler.title],
+ [difference.descriptionChanged, 'description_differs', 'meta description', browser.description, crawler.description],
+ [difference.h1Changed, 'h1_differs', 'H1 headings', browser.headings.h1, crawler.headings.h1],
+ ]) {
+ if (!changed) continue;
+ issues.push({
+ severity: 'warning',
+ code,
+ message: `The ${crawlerProfile.label} ${label} differs from the standard HTTP response.`,
+ why: `Different ${label} values may be intentional, personalized, or caused by crawler-specific response handling.`,
+ evidence: { standard: standardValue, crawler: crawlerValue },
+ nextStep: `Review both ${label} values and confirm that the difference is expected.`,
});
}
if (crawler.looksLikeAppShell) {
issues.push({
severity: 'critical',
code: 'crawler_app_shell',
- message: `The ${crawlerProfile.label} response appears to contain a thin JavaScript app shell.`,
+ message: `The ${crawlerProfile.label} response has limited visible content and multiple JavaScript app-shell signals.`,
+ why: 'The selected crawler may receive an application shell without the page’s primary content.',
+ evidence: {
+ readableCharacters: crawler.textLength,
+ scriptCount: crawler.scriptCount,
+ signals: crawler.appShellEvidence,
+ },
+ nextStep: 'Inspect the crawler HTTP response and verify whether primary content is present without JavaScript execution.',
+ });
+ }
+ if (materiallyDifferent) {
+ issues.push({
+ severity: 'warning',
+ code: 'crawler_response_differs',
+ message: `The ${crawlerProfile.label} response differs materially from the standard HTTP response.`,
+ why: 'This compatibility finding preserves the original pre-1.0 comparison code while specific findings explain each observed difference.',
+ evidence: {
+ statusChanged: browserResponse.statusCode !== crawlerResponse.statusCode,
+ textVolumeChanged: textVolumeDiffers,
+ titleChanged: difference.titleChanged,
+ descriptionChanged: difference.descriptionChanged,
+ h1Changed: difference.h1Changed,
+ },
+ nextStep: 'Review the specific comparison findings and confirm whether each difference is expected.',
+ compatibilityAlias: true,
});
}
@@ -60,6 +140,7 @@ export async function compareUrl(input, options = {}) {
checkedAt: new Date().toISOString(),
url,
crawlerProfile: { name: crawlerProfile.name, label: crawlerProfile.label },
+ comparisonMode: 'http-user-agent-responses',
browser: {
response: {
statusCode: browserResponse.statusCode,
@@ -77,12 +158,13 @@ export async function compareUrl(input, options = {}) {
html: crawler,
},
difference,
+ materiallyDifferent,
issues,
summary: issues.some((issue) => issue.severity === 'critical')
? 'critical'
: issues.length
? 'warning'
: 'pass',
- note: 'Different output is evidence to review, not proof of cloaking or a ranking problem.',
+ note: 'Both sides are HTTP responses. Neither executes JavaScript. Differences are evidence to review, not proof of cloaking or a ranking problem.',
};
}
diff --git a/src/discovery.js b/src/discovery.js
index 37fc02c..e3e2790 100644
--- a/src/discovery.js
+++ b/src/discovery.js
@@ -6,7 +6,7 @@ function fileUrl(siteUrl, pathname) {
return new URL(pathname, normalizePublicUrl(siteUrl)).toString();
}
-function parseRobots(text) {
+export function parseRobots(text) {
const sitemapLines = text
.split(/\r?\n/)
.map((line) => line.match(/^\s*sitemap\s*:\s*(\S+)\s*$/i)?.[1])
@@ -21,7 +21,7 @@ function parseRobots(text) {
return { sitemapLines, invalidSitemaps };
}
-function parseSitemap(text, expectedHostname) {
+export function parseSitemap(text, expectedHostname) {
const locations = [...text.matchAll(/]*>([\s\S]*?)<\/loc>/gi)]
.map((match) => match[1].trim());
const invalidUrls = [];
@@ -73,6 +73,8 @@ export async function checkDiscoveryFiles(input, options = {}) {
accept,
timeoutMs: options.timeoutMs,
maxChars: 1_000_000,
+ fetchFn: options.fetchFn,
+ assertUrlFn: options.assertUrlFn,
});
return [name, response];
}));
@@ -84,16 +86,32 @@ export async function checkDiscoveryFiles(input, options = {}) {
severity: name === 'llms.txt' ? 'warning' : 'critical',
code: 'http_error',
message: `${name} returned HTTP ${response.statusCode}.`,
+ why: `${name} could not be read successfully at its conventional public URL.`,
+ evidence: { statusCode: response.statusCode, finalUrl: response.finalUrl },
+ nextStep: `Confirm whether ${name} should exist and that its public URL returns the intended file.`,
});
}
if (name === 'robots.txt') {
const details = parseRobots(response.text);
+ if (response.ok && response.contentType && !/(?:text\/plain|text\/robots|application\/octet-stream)/i.test(response.contentType)) {
+ issues.push({
+ severity: 'warning',
+ code: 'unexpected_content_type',
+ message: `robots.txt returned ${response.contentType}.`,
+ why: 'An HTML fallback or unexpected media type can hide a missing robots.txt file.',
+ evidence: { contentType: response.contentType },
+ nextStep: 'Return robots.txt as plain text and verify that the route is not serving an HTML fallback.',
+ });
+ }
if (details.invalidSitemaps.length) {
issues.push({
severity: 'warning',
code: 'invalid_sitemap_directive',
message: 'One or more Sitemap directives are not valid absolute HTTP(S) URLs.',
+ why: 'Crawler sitemap directives should resolve without relying on a document base URL.',
+ evidence: { invalidValues: details.invalidSitemaps },
+ nextStep: 'Replace relative or malformed Sitemap values with absolute HTTP(S) URLs.',
});
}
return resultForFile(name, response, details, issues);
@@ -101,11 +119,24 @@ export async function checkDiscoveryFiles(input, options = {}) {
if (name === 'sitemap.xml') {
const details = parseSitemap(response.text, hostname);
+ if (response.ok && response.contentType && !/(?:application|text)\/(?:[a-z0-9.+-]*\+)?xml/i.test(response.contentType)) {
+ issues.push({
+ severity: 'warning',
+ code: 'unexpected_content_type',
+ message: `sitemap.xml returned ${response.contentType}.`,
+ why: 'An HTML fallback or unexpected media type can hide a missing XML sitemap.',
+ evidence: { contentType: response.contentType },
+ nextStep: 'Return sitemap.xml with an XML content type and verify that the route is not serving an HTML fallback.',
+ });
+ }
if (response.ok && details.locationCount === 0) {
issues.push({
severity: 'warning',
code: 'no_sitemap_urls',
message: 'No URLs were found in sitemap.xml.',
+ why: 'A sitemap without URL locations does not provide discoverable page entries.',
+ evidence: { locationCount: 0 },
+ nextStep: 'Add absolute page URLs or confirm that this is an intentionally empty sitemap index.',
});
}
if (details.invalidUrls.length) {
@@ -113,6 +144,9 @@ export async function checkDiscoveryFiles(input, options = {}) {
severity: 'warning',
code: 'invalid_sitemap_urls',
message: 'One or more sitemap entries are not valid absolute HTTP(S) URLs.',
+ why: 'Relative or malformed sitemap locations may not be interpreted consistently.',
+ evidence: { invalidValues: details.invalidUrls },
+ nextStep: 'Replace invalid values with absolute HTTP(S) URLs.',
});
}
if (details.otherHosts.length) {
@@ -120,6 +154,9 @@ export async function checkDiscoveryFiles(input, options = {}) {
severity: 'warning',
code: 'different_sitemap_host',
message: 'One or more sitemap entries use a different hostname.',
+ why: 'Cross-host entries may be intentional, but often indicate a staging or canonical-host mismatch.',
+ evidence: { expectedHostname: hostname, otherHostUrls: details.otherHosts },
+ nextStep: 'Confirm that every hostname is intentional and publicly canonical.',
});
}
return resultForFile(name, response, details, issues);
@@ -130,8 +167,34 @@ export async function checkDiscoveryFiles(input, options = {}) {
hasHeading: /^\s*#\s+\S/m.test(response.text),
hasLinks: /https?:\/\/\S+/i.test(response.text),
};
+ if (response.ok && response.contentType && !/(?:text\/plain|text\/markdown)/i.test(response.contentType)) {
+ issues.push({
+ severity: 'warning',
+ code: 'unexpected_content_type',
+ message: `llms.txt returned ${response.contentType}.`,
+ why: 'An HTML fallback or unexpected media type can hide a missing llms.txt file.',
+ evidence: { contentType: response.contentType },
+ nextStep: 'Return llms.txt as plain text or Markdown and verify that the route is not serving an HTML fallback.',
+ });
+ }
if (response.ok && !response.text.trim()) {
- issues.push({ severity: 'warning', code: 'empty_llms', message: 'llms.txt is empty.' });
+ issues.push({
+ severity: 'warning',
+ code: 'empty_llms',
+ message: 'llms.txt is empty.',
+ why: 'An empty file provides no project summary or resource references.',
+ evidence: { characterCount: 0 },
+ nextStep: 'Add useful plain-text or Markdown content, or remove the empty file if it is not used.',
+ });
+ } else if (response.ok && !details.hasHeading) {
+ issues.push({
+ severity: 'warning',
+ code: 'llms_missing_heading',
+ message: 'llms.txt does not contain a Markdown H1 heading.',
+ why: 'A primary heading is a basic structural signal for the proposed llms.txt format.',
+ evidence: { hasHeading: false, characterCount: details.characterCount },
+ nextStep: 'Add one clear Markdown H1 heading near the beginning of the file.',
+ });
}
return resultForFile(name, response, details, issues);
});
diff --git a/src/fetch-public.js b/src/fetch-public.js
index 6a2bcf3..f25a1c4 100644
--- a/src/fetch-public.js
+++ b/src/fetch-public.js
@@ -4,7 +4,14 @@ const REDIRECT_CODES = new Set([301, 302, 303, 307, 308]);
const DEFAULT_MAX_CHARS = 500_000;
export async function readBoundedText(response, maxChars = DEFAULT_MAX_CHARS) {
- if (!response.body) return (await response.text()).slice(0, maxChars);
+ return (await readBoundedResult(response, maxChars)).text;
+}
+
+async function readBoundedResult(response, maxChars = DEFAULT_MAX_CHARS) {
+ if (!response.body) {
+ const text = await response.text();
+ return { text: text.slice(0, maxChars), truncated: text.length > maxChars };
+ }
const decoder = new TextDecoder();
const reader = response.body.getReader();
@@ -12,7 +19,7 @@ export async function readBoundedText(response, maxChars = DEFAULT_MAX_CHARS) {
let completed = false;
try {
- while (output.length < maxChars) {
+ while (output.length <= maxChars) {
const { done, value } = await reader.read();
if (done) {
completed = true;
@@ -21,7 +28,7 @@ export async function readBoundedText(response, maxChars = DEFAULT_MAX_CHARS) {
output += decoder.decode(value, { stream: true });
}
output += decoder.decode();
- return output.slice(0, maxChars);
+ return { text: output.slice(0, maxChars), truncated: output.length > maxChars };
} finally {
if (!completed) await reader.cancel().catch(() => {});
reader.releaseLock();
@@ -64,13 +71,16 @@ export async function fetchPublicText(target, options = {}) {
}
if (!REDIRECT_CODES.has(response.status)) {
+ const body = await readBoundedResult(response, maxChars);
return {
requestedUrl: normalizePublicUrl(target),
finalUrl: currentUrl,
statusCode: response.status,
ok: response.ok,
contentType: response.headers.get('content-type') || '',
- text: await readBoundedText(response, maxChars),
+ text: body.text,
+ truncated: body.truncated,
+ maxChars,
};
}
@@ -83,6 +93,8 @@ export async function fetchPublicText(target, options = {}) {
ok: response.ok,
contentType: response.headers.get('content-type') || '',
text: '',
+ truncated: false,
+ maxChars,
};
}
if (redirects === maxRedirects) throw new Error('Too many redirects while checking this URL.');
diff --git a/src/format.js b/src/format.js
index 77b0e4a..2bc3910 100644
--- a/src/format.js
+++ b/src/format.js
@@ -2,9 +2,24 @@ function line(label, value) {
return `${label.padEnd(18)} ${value ?? '—'}`;
}
+function evidence(value) {
+ if (value === undefined) return '';
+ return typeof value === 'string' ? value : JSON.stringify(value);
+}
+
+function formatIssue(issue) {
+ return [
+ `- ${issue.severity.toUpperCase()} [${issue.code}]: ${issue.message}`,
+ issue.why ? ` Why: ${issue.why}` : null,
+ issue.evidence !== undefined ? ` Evidence: ${evidence(issue.evidence)}` : null,
+ issue.nextStep ? ` Next: ${issue.nextStep}` : null,
+ ].filter(Boolean).join('\n');
+}
+
function formatIssues(issues) {
if (!issues.length) return '\nNo material issues detected by this check.';
- return `\nIssues:\n${issues.map((issue) => `- ${issue.severity.toUpperCase()}: ${issue.message}`).join('\n')}`;
+ const visibleIssues = issues.filter((issue) => !issue.compatibilityAlias);
+ return `\nIssues:\n${visibleIssues.map(formatIssue).join('\n')}`;
}
export function formatHuman(result) {
@@ -30,12 +45,17 @@ export function formatHuman(result) {
`Prerender Buddy · response comparison · ${result.summary.toUpperCase()}`,
line('URL', result.url),
line('Crawler profile', result.crawlerProfile.label),
- line('Browser HTTP', result.browser.response.statusCode),
+ line('Standard HTTP', result.browser.response.statusCode),
line('Crawler HTTP', result.crawler.response.statusCode),
- line('Browser text', `${result.browser.html.textLength} characters`),
+ line('Standard text', `${result.browser.html.textLength} characters`),
line('Crawler text', `${result.crawler.html.textLength} characters`),
line('Text ratio', result.difference.textRatio),
+ line(
+ 'Accepted ratio',
+ `${result.difference.acceptedTextRatio.minimum}–${result.difference.acceptedTextRatio.maximum}`,
+ ),
line('Title changed', result.difference.titleChanged ? 'yes' : 'no'),
+ line('Description changed', result.difference.descriptionChanged ? 'yes' : 'no'),
line('H1 changed', result.difference.h1Changed ? 'yes' : 'no'),
formatIssues(result.issues),
`\n${result.note}`,
@@ -46,7 +66,7 @@ export function formatHuman(result) {
`\n${file.name} · ${file.summary.toUpperCase()}`,
line('URL', file.url),
line('HTTP', file.statusCode),
- ...file.issues.map((issue) => `- ${issue.severity.toUpperCase()}: ${issue.message}`),
+ ...file.issues.map(formatIssue),
]);
return [
`Prerender Buddy · discovery-file check · ${result.summary.toUpperCase()}`,
diff --git a/src/html.js b/src/html.js
index d5ac3f5..872a3ce 100644
--- a/src/html.js
+++ b/src/html.js
@@ -15,6 +15,7 @@ function cleanText(value = '') {
export function stripTags(html = '') {
return cleanText(html
.replace(//g, ' ')
+ .replace(//gi, ' ')
.replace(/<(script|style|noscript|template)\b[\s\S]*?<\/\1>/gi, ' ')
.replace(/<[^>]+>/g, ' '));
}
@@ -70,6 +71,22 @@ export function analyzeHtml(html = '') {
const scriptCount = (html.match(/]*\bid=["'](?:root|app)["'][^>]*>\s*<\/(?:div|main)>/i.test(html);
+ const hasLoadingPlaceholder = textLength < 180
+ && /\b(?:loading|please wait|initializing|starting)\b/i.test(visibleText);
+ const hasModuleOrBundledScript = /]*(?:type=["']module["']|src=["'][^"']*(?:\/assets\/|bundle|app)[^"']*\.js)/i.test(html);
+ const appShellEvidence = [
+ ...(hasEmptyMountPoint ? ['empty root or app mount point'] : []),
+ ...(hasLoadingPlaceholder ? ['loading-only visible text'] : []),
+ ...(hasModuleOrBundledScript ? ['module or bundled application script'] : []),
+ ...(scriptCount >= 2 ? [`${scriptCount} script elements`] : []),
+ ...signs.map((sign) => `${sign} detected`),
+ ];
+ const looksLikeAppShell = textLength < 300 && (
+ (hasEmptyMountPoint && scriptCount >= 1)
+ || (hasLoadingPlaceholder && hasModuleOrBundledScript)
+ || (textLength < 80 && hasModuleOrBundledScript)
+ );
return {
title: tagContent(html, /]*>([\s\S]*?)<\/title>/i),
@@ -86,35 +103,112 @@ export function analyzeHtml(html = '') {
textExcerpt: visibleText.slice(0, 500),
scriptCount,
frameworkSigns: signs,
- looksLikeAppShell: textLength < 300 && (scriptCount >= 2 || signs.length > 0),
+ appShellEvidence,
+ looksLikeAppShell,
};
}
export function buildHtmlIssues(summary, response = {}) {
const issues = [];
if (!response.ok) {
- issues.push({ severity: 'critical', code: 'http_error', message: `Page returned HTTP ${response.statusCode}.` });
+ issues.push({
+ severity: 'critical',
+ code: 'http_error',
+ message: `Page returned HTTP ${response.statusCode}.`,
+ why: 'An unsuccessful HTTP response can prevent crawlers from accessing the page content.',
+ evidence: { statusCode: response.statusCode },
+ nextStep: 'Confirm that the public URL returns a successful response for the selected crawler profile.',
+ });
+ }
+ if (response.contentType && !/(?:text\/html|application\/xhtml\+xml)/i.test(response.contentType)) {
+ issues.push({
+ severity: 'critical',
+ code: 'unexpected_content_type',
+ message: `Page returned ${response.contentType} instead of HTML.`,
+ why: 'HTML diagnostics are not reliable when the response declares a different media type.',
+ evidence: { contentType: response.contentType },
+ nextStep: 'Check the requested route and its Content-Type header.',
+ });
+ }
+ if (response.truncated) {
+ issues.push({
+ severity: 'warning',
+ code: 'response_truncated',
+ message: `Analysis stopped after the configured ${response.maxChars} character response limit.`,
+ why: 'Signals after the response limit were not analysed.',
+ evidence: { maxChars: response.maxChars },
+ nextStep: 'Review the response size and rerun with a focused page when possible.',
+ });
}
if (!summary.title) {
- issues.push({ severity: 'warning', code: 'missing_title', message: 'Raw HTML is missing a page title.' });
+ issues.push({
+ severity: 'warning',
+ code: 'missing_title',
+ message: 'Returned HTML is missing a page title.',
+ why: 'The title is a primary page-identification signal in the returned HTML.',
+ evidence: { title: '' },
+ nextStep: 'Add a descriptive to the initial HTML response.',
+ });
}
if (!summary.description) {
- issues.push({ severity: 'warning', code: 'missing_description', message: 'Raw HTML is missing a meta description.' });
+ issues.push({
+ severity: 'warning',
+ code: 'missing_description',
+ message: 'Returned HTML is missing a meta description.',
+ why: 'A description helps crawlers and preview systems understand the page summary.',
+ evidence: { description: '' },
+ nextStep: 'Add a page-specific meta description to the initial HTML response.',
+ });
}
if (!summary.headings.h1.length) {
- issues.push({ severity: 'warning', code: 'missing_h1', message: 'Raw HTML is missing an H1 heading.' });
+ issues.push({
+ severity: 'warning',
+ code: 'missing_h1',
+ message: 'Returned HTML is missing an H1 heading.',
+ why: 'A primary heading provides a clear content label in the returned document.',
+ evidence: { h1Count: 0 },
+ nextStep: 'Include the page’s primary heading in the initial HTML response.',
+ });
+ }
+ if (summary.canonicalUrl) {
+ let canonicalIsValid = false;
+ try {
+ canonicalIsValid = ['http:', 'https:'].includes(new URL(summary.canonicalUrl).protocol);
+ } catch {
+ canonicalIsValid = false;
+ }
+ if (!canonicalIsValid) {
+ issues.push({
+ severity: 'warning',
+ code: 'invalid_canonical',
+ message: 'Returned HTML contains a canonical URL that is not an absolute HTTP(S) URL.',
+ why: 'A malformed or relative canonical can make the preferred page URL ambiguous.',
+ evidence: { canonicalUrl: summary.canonicalUrl },
+ nextStep: 'Replace the canonical value with the intended absolute public HTTP(S) URL.',
+ });
+ }
}
if (summary.looksLikeAppShell) {
issues.push({
severity: 'critical',
code: 'app_shell',
- message: 'Raw HTML has limited visible content and JavaScript app-shell signs.',
+ message: 'Returned HTML has limited visible content and multiple JavaScript app-shell signals.',
+ why: 'Crawlers that do not execute JavaScript may receive only the application shell.',
+ evidence: {
+ readableCharacters: summary.textLength,
+ scriptCount: summary.scriptCount,
+ signals: summary.appShellEvidence,
+ },
+ nextStep: 'Inspect the raw response and test whether important page content is present before JavaScript executes.',
});
} else if (summary.textLength < 300) {
issues.push({
severity: 'warning',
code: 'thin_html',
- message: 'Raw HTML contains less than 300 readable characters.',
+ message: 'Returned HTML contains less than 300 readable characters.',
+ why: 'A short response may be legitimate, but it may also omit important page content.',
+ evidence: { readableCharacters: summary.textLength, threshold: 300 },
+ nextStep: 'Review whether the returned text contains the page’s primary information.',
});
}
return issues;
diff --git a/src/index.js b/src/index.js
index a3a78ef..eef1df3 100644
--- a/src/index.js
+++ b/src/index.js
@@ -1,6 +1,7 @@
export { checkUrl } from './check.js';
-export { compareUrl } from './compare.js';
-export { checkDiscoveryFiles } from './discovery.js';
+export { compareUrl, contentDelta } from './compare.js';
+export { checkDiscoveryFiles, parseRobots, parseSitemap } from './discovery.js';
+export { formatHuman } from './format.js';
export { analyzeHtml, buildHtmlIssues, stripTags } from './html.js';
export { USER_AGENT_PROFILES, getUserAgentProfile } from './profiles.js';
export { assertPublicUrl, isBlockedIp, normalizePublicUrl } from './url-safety.js';
diff --git a/src/profiles.js b/src/profiles.js
index f503cbd..aea0110 100644
--- a/src/profiles.js
+++ b/src/profiles.js
@@ -1,6 +1,6 @@
export const USER_AGENT_PROFILES = Object.freeze({
browser: {
- label: 'Browser',
+ label: 'Browser-style user agent',
value: 'Mozilla/5.0 (compatible; PrerenderBuddyCLI/0.1; +https://prerenderbuddy.com)',
},
googlebot: {
diff --git a/test/check.test.js b/test/check.test.js
new file mode 100644
index 0000000..fcc0b9f
--- /dev/null
+++ b/test/check.test.js
@@ -0,0 +1,53 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import test from 'node:test';
+import { checkUrl } from '../src/check.js';
+import { formatHuman } from '../src/format.js';
+
+async function fixture(name) {
+ return readFile(new URL(`./fixtures/html/${name}`, import.meta.url), 'utf8');
+}
+
+test('checks returned HTML with the selected crawler profile', async () => {
+ const result = await checkUrl('https://example.com/page', {
+ userAgent: 'gptbot',
+ assertUrlFn: async () => {},
+ fetchFn: async () => new Response(await fixture('healthy.html'), {
+ status: 200,
+ headers: { 'content-type': 'text/html; charset=utf-8' },
+ }),
+ });
+
+ assert.equal(result.command, 'check');
+ assert.equal(result.profile.name, 'gptbot');
+ assert.equal(result.summary, 'pass');
+ assert.equal(result.response.truncated, false);
+});
+
+test('separates HTTP and content-type failures from heuristic findings', async () => {
+ const result = await checkUrl('https://example.com/data', {
+ assertUrlFn: async () => {},
+ fetchFn: async () => new Response('{"message":"blocked"}', {
+ status: 403,
+ headers: { 'content-type': 'application/json' },
+ }),
+ });
+
+ assert.equal(result.summary, 'critical');
+ assert.ok(result.issues.some((issue) => issue.code === 'http_error'));
+ assert.ok(result.issues.some((issue) => issue.code === 'unexpected_content_type'));
+});
+
+test('README demonstration matches actual formatter output', async () => {
+ const html = await fixture('loading-placeholder.html');
+ const result = await checkUrl('https://example.com/app', {
+ userAgent: 'googlebot',
+ assertUrlFn: async () => {},
+ fetchFn: async () => new Response(html, {
+ status: 200,
+ headers: { 'content-type': 'text/html; charset=utf-8' },
+ }),
+ });
+ const readme = await readFile(new URL('../README.md', import.meta.url), 'utf8');
+ assert.ok(readme.includes(formatHuman(result)));
+});
diff --git a/test/cli.test.js b/test/cli.test.js
index 310149e..55d0142 100644
--- a/test/cli.test.js
+++ b/test/cli.test.js
@@ -1,6 +1,13 @@
import assert from 'node:assert/strict';
+import { spawnSync } from 'node:child_process';
import test from 'node:test';
-import { parseArgs, shouldFail } from '../src/cli.js';
+import { fileURLToPath } from 'node:url';
+import {
+ executionErrorResult,
+ parseArgs,
+ runCli,
+ shouldFail,
+} from '../src/cli.js';
test('parses common CI options', () => {
const result = parseArgs([
@@ -13,6 +20,8 @@ test('parses common CI options', () => {
'--json',
'--fail-on',
'critical',
+ '--text-ratio-threshold',
+ '0.2',
]);
assert.deepEqual(result.positional, ['check', 'https://example.com']);
@@ -20,11 +29,16 @@ test('parses common CI options', () => {
assert.equal(result.options.timeoutMs, 5000);
assert.equal(result.options.json, true);
assert.equal(result.options.failOn, 'critical');
+ assert.equal(result.options.textRatioThreshold, 0.2);
});
test('rejects unsafe timeout and failure threshold values', () => {
assert.throws(() => parseArgs(['check', 'example.com', '--timeout', '10']), /between 1000 and 60000/);
assert.throws(() => parseArgs(['check', 'example.com', '--fail-on', 'pass']), /warning or critical/);
+ assert.throws(() => parseArgs(['check', 'example.com', '--text-ratio-threshold', '2']), /between 0.01 and 0.99/);
+ assert.throws(() => parseArgs(['check', 'example.com', '--user-agent']), /requires a value/);
+ assert.throws(() => parseArgs(['check', 'example.com', '--user-agent', 'unknown']), /Unknown user-agent/);
+ assert.throws(() => parseArgs(['check', 'example.com', '--unknown']), /Unknown option/);
});
test('maps summaries to CI thresholds', () => {
@@ -33,3 +47,97 @@ test('maps summaries to CI thresholds', () => {
assert.equal(shouldFail('warning', 'warning'), true);
assert.equal(shouldFail('critical', 'critical'), true);
});
+
+test('prints help and version without running a command', async () => {
+ const output = [];
+ await runCli(['--help'], { write: (value) => output.push(value) });
+ assert.match(output.join(''), /Usage:/);
+ assert.match(output.join(''), /text-ratio-threshold/);
+
+ output.length = 0;
+ await runCli(['--version'], { write: (value) => output.push(value) });
+ assert.match(output.join(''), /^\d+\.\d+\.\d+\n$/);
+});
+
+test('runs every command in JSON mode without human-readable output', async () => {
+ for (const command of ['check', 'compare', 'files']) {
+ const output = [];
+ const exitCodes = [];
+ await runCli([command, 'https://example.com', '--json', '--fail-on', 'warning'], {
+ write: (value) => output.push(value),
+ setExitCode: (value) => exitCodes.push(value),
+ handlers: {
+ [command]: async () => ({ command, summary: 'warning', issues: [] }),
+ },
+ });
+ const parsed = JSON.parse(output.join(''));
+ assert.equal(parsed.command, command);
+ assert.deepEqual(exitCodes, [1]);
+ }
+});
+
+test('clean checks leave the process exit code unchanged', async () => {
+ const exitCodes = [];
+ await runCli(['check', 'https://example.com', '--json', '--fail-on', 'critical'], {
+ write: () => {},
+ setExitCode: (value) => exitCodes.push(value),
+ handlers: {
+ check: async () => ({ command: 'check', summary: 'pass' }),
+ },
+ });
+ assert.deepEqual(exitCodes, []);
+});
+
+test('passes compare thresholds to the command handler', async () => {
+ let received;
+ await runCli(['compare', 'https://example.com', '--json', '--text-ratio-threshold', '0.15'], {
+ write: () => {},
+ handlers: {
+ compare: async (_url, options) => {
+ received = options;
+ return { command: 'compare', summary: 'pass' };
+ },
+ },
+ });
+ assert.equal(received.textRatioThreshold, 0.15);
+});
+
+test('rejects invalid commands, missing URLs, and extra positional arguments', async () => {
+ await assert.rejects(() => runCli(['unknown', 'https://example.com']), /Unknown command/);
+ await assert.rejects(() => runCli(['check']), /requires a URL/);
+ await assert.rejects(() => runCli(['check', 'https://example.com', 'extra']), /Only one URL/);
+});
+
+test('classifies execution errors for machine-readable output', () => {
+ assert.equal(executionErrorResult(new Error('Request timed out after 1000 ms.')).error.code, 'timeout');
+ assert.equal(executionErrorResult(new Error('The URL resolves to a private or blocked network address.')).error.code, 'unsafe_target');
+ assert.equal(executionErrorResult(new Error('Unknown command "bad".')).error.code, 'invalid_input');
+ assert.equal(executionErrorResult(new Error('socket closed')).error.code, 'request_failed');
+});
+
+test('the executable returns JSON-only output and exit code 2 for invalid JSON-mode input', () => {
+ const result = spawnSync(process.execPath, [
+ fileURLToPath(new URL('../bin/prerenderbuddy.js', import.meta.url)),
+ 'check',
+ 'file:///etc/passwd',
+ '--json',
+ ], { encoding: 'utf8' });
+
+ assert.equal(result.status, 2);
+ assert.equal(result.stderr, '');
+ const parsed = JSON.parse(result.stdout);
+ assert.equal(parsed.summary, 'error');
+ assert.equal(parsed.error.code, 'unsafe_target');
+});
+
+test('the executable keeps human execution errors on stderr', () => {
+ const result = spawnSync(process.execPath, [
+ fileURLToPath(new URL('../bin/prerenderbuddy.js', import.meta.url)),
+ 'unknown',
+ 'https://example.com',
+ ], { encoding: 'utf8' });
+
+ assert.equal(result.status, 2);
+ assert.equal(result.stdout, '');
+ assert.match(result.stderr, /Prerender Buddy check failed: Unknown command/);
+});
diff --git a/test/compare.test.js b/test/compare.test.js
new file mode 100644
index 0000000..72d14b7
--- /dev/null
+++ b/test/compare.test.js
@@ -0,0 +1,78 @@
+import assert from 'node:assert/strict';
+import { readFile } from 'node:fs/promises';
+import test from 'node:test';
+import { compareUrl, contentDelta } from '../src/compare.js';
+import { analyzeHtml } from '../src/html.js';
+
+async function fixture(name) {
+ return readFile(new URL(`./fixtures/html/${name}`, import.meta.url), 'utf8');
+}
+
+function responseQueue(...responses) {
+ return async () => responses.shift();
+}
+
+test('compares two HTTP user-agent responses without claiming browser rendering', async () => {
+ const html = await fixture('healthy.html');
+ const result = await compareUrl('https://example.com', {
+ userAgent: 'gptbot',
+ assertUrlFn: async () => {},
+ fetchFn: responseQueue(
+ new Response(html, { status: 200, headers: { 'content-type': 'text/html' } }),
+ new Response(html, { status: 200, headers: { 'content-type': 'text/html' } }),
+ ),
+ });
+
+ assert.equal(result.comparisonMode, 'http-user-agent-responses');
+ assert.equal(result.summary, 'pass');
+ assert.match(result.note, /Neither executes JavaScript/);
+});
+
+test('reports exact metadata and text-volume differences separately', async () => {
+ const standard = await fixture('healthy.html');
+ const crawler = '<html><head><title>DifferentOther H1
Short