diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a94cf98..1af830f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,6 +7,9 @@ on: branches: [main] workflow_dispatch: +permissions: + contents: read + jobs: lint-test-build: name: Lint · Test · Build @@ -46,3 +49,69 @@ jobs: - name: Build run: npm run build + + secret-scan: + name: Secret scan + runs-on: ubuntu-latest + # A normal run takes seconds. The cap keeps a pathological file from holding a runner for the default 6 hours. + timeout-minutes: 5 + + permissions: + contents: read + + steps: + # Full history (not the default depth 1): the range scan below needs the base and head commits of the change and + # everything between them. No credentials are kept in the checkout, since nothing here pushes. + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 20 + + # Scans git-tracked files only. The report lists file, line and rule name, never the matched text. + # The --history mode is deliberately not run here: the known leak in old commits would fail forever. + - name: Scan tracked files for secrets + run: node scripts/check-secrets.mjs + + # The tip scan cannot see a secret that was committed and then removed by a later commit of the same pull request or + # push, and that value is still in the pushed history. This step scans the ADDED lines of every commit in the range: + # pull_request base.sha..head.sha (commits reachable from the PR head and not from its base; a fork's head commit + # arrives through the PR merge commit that checkout fetches, and is present with fetch-depth 0) + # push before..sha (a new branch has an all-zero "before": only the pushed tip commit is scanned) + # A commit that is not in the clone (a force-push removed the old tip, a fetch was partial) or a shallow clone fails the + # step with exit code 2 and a message. It never passes silently and never falls back to the full-history audit. + # Event values reach the script only through env vars, are checked to be plain commit ids, and are quoted. + - name: Scan commits in this change for secrets + env: + EVENT_NAME: ${{ github.event_name }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + PUSH_BEFORE_SHA: ${{ github.event.before }} + PUSH_AFTER_SHA: ${{ github.sha }} + run: | + set -euo pipefail + case "$EVENT_NAME" in + pull_request) + base="$PR_BASE_SHA" + head="$PR_HEAD_SHA" + ;; + push) + base="$PUSH_BEFORE_SHA" + head="$PUSH_AFTER_SHA" + ;; + *) + echo "No commit range for a '$EVENT_NAME' run: only the tracked-file scan above applies." + exit 0 + ;; + esac + id_pattern='^([0-9a-f]{40}|[0-9a-f]{64})$' + if ! [[ "$base" =~ $id_pattern && "$head" =~ $id_pattern ]]; then + echo "::error::The event did not give plain base and head commit ids, so the commits in this change cannot be scanned." + exit 2 + fi + node scripts/check-secrets.mjs --range "$base..$head" diff --git a/CLAUDE.md b/CLAUDE.md index e855592..df1fb75 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,8 +47,9 @@ cd app && npm run build ```bash cd app && npm install cd ../server && npm install -# Copy and configure env files (no quotes around values — Vite includes them literally) -cp app/.env.example app/.env.development +# Create your untracked local env file (no quotes around values — Vite includes them literally). +# Never overwrite the tracked app/.env.development or app/.env.production; see SECURITY_NOTICE.md. +[ -e app/.env.development.local ] || cp app/.env.example app/.env.development.local ``` ## Architecture diff --git a/SECURITY_NOTICE.md b/SECURITY_NOTICE.md index 87e8651..c84026d 100644 --- a/SECURITY_NOTICE.md +++ b/SECURITY_NOTICE.md @@ -1,85 +1,140 @@ -# Security Notice: Environment Variable Cleanup +# Security Notice: Credential Exposure and Rotation -## Actions Taken +> **Status: OPEN. Rotation is NOT recorded as done.** +> +> Real credentials were committed to this public repository in December 2025 and are still readable in its git history. This notice stays open until the repository owner completes the checklist below, fills in every "done on" date and records the decision on git history (step 5.1). Nothing in this file says a credential has been rotated, revoked or confirmed dead. Until it does, treat the Neon database password and the Stack Auth secret server key as compromised. +> +> Tracking issue: #22. Only the owner can do the rotation (it needs the Neon, Stack Auth and Vercel dashboards). An agent or CI job cannot, and must never be given the old or new values. -The following files have been secured by replacing real credentials with placeholder values: +## What leaked -### 1. `app/.env.development` -- **Status**: Tracked in git, now contains only placeholders -- **Local secrets moved to**: `app/.env.development.local` (gitignored) +Found by scanning every commit on every ref of the public repository (356 commits, 49 branches, 73 pull request refs, 1 tag). Values are deliberately not reproduced anywhere in this file. -### 2. `app/.env.production` -- **Status**: Tracked in git, now contains only placeholders -- **Local secrets moved to**: `app/.env.production.local` (gitignored) +| Service | What | Secret? | Files | Introduced / removed from `main` | Action | +|---|---|---|---|---|---| +| Neon Postgres | Connection string containing the owner-role password | **Yes** | `app/.env.development`, `app/.env.production` | `1eef6e2` (2025-12-16) / `0d36e57` (2026-03-28) | Rotate: step 1 | +| Stack Auth (retired) | Secret server key | **Yes** | same two files | same commits | Revoke: step 2 | +| Stack Auth (retired) | Project ID and publishable client key | No, shipped to browsers by design | same two files, plus `docs/ENVIRONMENT_VARIABLES.md` | `1eef6e2` / files `0d36e57`, docs `32f7f12` (2026-08-06) | None; moot once the Stack project is revoked | +| Legacy JWT login | Weak hardcoded fallback for `JWT_SECRET` in `server/server.js` (a guessable dev default, not a random production secret). The tag `v1.0.0` points at the introducing commit `9156730`, so it carries the fallback at its tip | Weak | `server/server.js` | `9156730` (2025-12-16) / `0d36e57` | Check the real value: step 3 | +| Firebase | Web config. Only a project ID was ever committed | No, public by design | `docs/ENVIRONMENT_VARIABLES.md` | n/a | None | -### Already Secure Files -- `.env.local` - Already gitignored, contains Vercel-generated credentials -- `app/.env` - Already gitignored, contains real credentials -- `server/.env` - Already gitignored, contains configuration +Also found and harmless: test constants and a synthetic test token in the test files, a dummy bcrypt hash in `shared/handlers/login.js`, and a truncated JWT header sample in `docs/API.md`. No AWS, GitHub, Slack, Stripe, Google or Gemini API key, private key, service-account file or full JWT was ever committed. -## Exposed Credentials +**How exposed.** The repository is public. On `main` the values were present from 2025-12-16 to 2026-03-28 (about 3.4 months). When scanned they were still retrievable: every branch contained the introducing commit, the values were at the tip of 8 branches (`docs/add-changelog`, `feature/custom-dropdown-options`, `feature/import-validation-ux`, `feature/pcs-maritime-ui-overhaul`, `fix/docs-env-vars`, `fix/neon-tuna-pdf-import`, `fix/oauth-authentication-config`, `vercel/set-up-vercel-web-analytics-in-qwrrs8`) and they appeared in 12 pull request refs (PRs 1 to 10, 12 and 73). Assume they were copied. The weak JWT fallback is also at the tip of the tag `v1.0.0` (the env files are not in that tag). -The following credentials were previously committed to the repository and **MUST be rotated immediately**: +**Current tree is clean.** At the current branch tip no tracked file contains a real credential; the four tracked env files hold placeholders only. Earlier notes (for example `AUDIT_REPORT.md` and the comments on #22) may name a different first commit, such as the oldest commit visible in a shallow clone. A shallow clone only holds the newest commits, so `git log` there will not show `1eef6e2`. -### Previous Auth Provider Credentials -- **Project ID / Client Keys**: [REDACTED - rotate immediately] -- **Secret Server Key**: [REDACTED - rotate immediately] -- **Action Required**: Revoke the retired Stack Auth credentials if they still exist, and configure Firebase Auth credentials for the active deployment. +## Rotation checklist -### Neon Database -- **Database URL**: [REDACTED - reset password immediately] -- **Username**: [REDACTED - reset password immediately] -- **Password**: [REDACTED - reset password immediately] -- **Action Required**: Reset database password in Neon dashboard at https://console.neon.tech +Do these in order (step 1 is staged: new credential live before the old one is revoked). Record the date you finish each one. Do not paste old or new values into issues, PRs, chat, tickets or an AI assistant. -## Immediate Action Items +### 1. Neon database (first, it is the most serious) -1. **Revoke Retired Auth Credentials** - - Revoke any retired Stack Auth project keys - - Configure Firebase Auth for the active project - - Update `app/.env.development.local` and `app/.env.production.local` - - Update Vercel environment variables +Do this as a **staged rotation**: bring the app up on a new credential first, and only then kill the compromised one. That keeps production up throughout. -2. **Rotate Neon Database Password** - - Go to Neon console - - Reset the database owner password - - Update all `.env*.local` files with new connection string - - Update Vercel environment variables - - Update `.env.local` file +The faster alternative is to reset the existing role's password in place (Neon console, the role's "Reset password"). It is simpler but accepts an outage: the moment the password changes, every new database connection from the deployed site fails with a password authentication error (API routes return 500 and sign-in that needs the database, saved calculations and data pages break) until steps 1.2 and 1.3 finish. Connections already open may keep working for a while, which can hide the problem. If you take that route, do 1.2 to 1.4 immediately after the reset, then continue with 1.6 onward. -3. **Update Vercel Environment Variables** - - Go to Vercel dashboard → Settings → Environment Variables - - Update all rotated credentials - - Redeploy the application +- [ ] 1.1 In the [Neon console](https://console.neon.tech), create a **new** role (new name, new password) that can do what the API needs, and copy its connection string into your password manager. If the old role owns the database or its tables, you will move ownership in 1.5. Do not touch the compromised role yet. done on: ____ +- [ ] 1.2 Set the new connection string as `DATABASE_URL` in Vercel for **Production, Preview and Development**. The Neon/Vercel integration may also have added variables holding the old password (`POSTGRES_*`, `PG*`, `DATABASE_URL_UNPOOLED`, `NEON_*`). Nothing in this repository reads them, so update or delete them. Remove stale duplicate Neon integrations (see `DEPLOYMENT.md` section 5.3). done on: ____ +- [ ] 1.3 Redeploy Production (and any Preview you still use). A variable change only reaches new deployments. Older deployments keep the old value and will lose database access once the old role is revoked, so delete the ones you no longer need. This also limits rollback: once the old role is revoked in 1.5, do not roll back or promote a deployment built before this step, because it still carries the old connection string. Roll back only to a deployment built after 1.3. done on: ____ +- [ ] 1.4 Smoke test the deployed site on the new role: sign in with Firebase email/password, sign in with Google, sign in with a legacy username, and open a page that reads or saves data (for example Saved calculations). Check the Vercel function logs for database authentication errors. Do not continue until this passes. done on: ____ +- [ ] 1.5 Revoke the compromised role. First move what it owns to the new role (for example `REASSIGN OWNED BY TO ;` run as an owner, and change the database owner if it was the old role), then delete the old role, or at minimum reset its password to a fresh random value that you discard. If you took the reset-in-place shortcut, this step is already done by the reset. done on: ____ +- [ ] 1.6 Check every Neon branch of the project, especially any created before 1.5. A branch forked from production may still carry the old role and password. Reset or delete the role there too, or delete the branch. done on: ____ +- [ ] 1.7 **Confirm the old credential is dead, yourself.** From your own machine, try to connect with the old connection string (for example `psql` with the old string) and confirm authentication is refused. The old string is in your own records or in commit `1eef6e2`; read it only in your own terminal. Do not give it to a script, a CI job or an agent, and clear it from your shell history afterwards. done on: ____ +- [ ] 1.8 Review Neon's connection and query history for unexpected clients or queries between 2025-12-16 and the date you completed 1.5. What is available depends on your Neon plan. Note what you found: ____ done on: ____ +- [ ] 1.9 Replace local copies: re-run `vercel env pull` to refresh any root `.env.local`, and clean any shell profile or password manager entry that still holds the old string. done on: ____ -4. **Clean Git History (Optional but Recommended)** - - Consider using tools like `git filter-branch` or `BFG Repo-Cleaner` to remove exposed secrets from git history - - Note: This requires force-pushing and coordinating with all repository collaborators +Optional hardening: run the API as a role with only the table access it needs, and keep the owner credential for the schema and import scripts in `scripts/`. Today the API and the scripts use the same owner-level string. -## For New Developers +### 2. Stack Auth (retired provider) -When setting up this project locally: +Nothing in `api/`, `app/src`, `server/`, `shared/` or `scripts/` reads a `STACK_*` variable any more. Auth moved to Firebase in PR #63 (`32f7f12`, 2026-08-06), so revoking the key has no effect on production as long as production runs that commit or a later one. -1. Copy template files to create local environment files: - ```bash - cp app/.env.development app/.env.development.local - cp app/.env.production app/.env.production.local - ``` +- [ ] 2.1 In the Stack Auth dashboard, revoke the secret server key, or delete the retired project. If the project was created through the Neon Auth integration, look in the Neon console's Auth section instead. done on: ____ +- [ ] 2.2 Remove every `STACK_*`, `NEXT_PUBLIC_STACK_*` and `VITE_STACK_*` variable from Vercel (all environments). `DEPLOYMENT.md` section 5.3 covers removing the integration but does not say to revoke the key at the provider, so 2.1 is still needed. done on: ____ -2. Request the real credentials from a team lead and update the `.local` files +The project ID and publishable client key need no separate action. -3. **Never commit `.env*.local` files** - they are gitignored for security +### 3. `JWT_SECRET` (legacy `/api/login`) -## Files Currently Gitignored +No real `JWT_SECRET` value was ever committed. The only exposure is the weak fallback that existed in an old `server/server.js`, and it matters only if a deployment ran that server without `JWT_SECRET` set. -The following patterns are in `.gitignore` to protect credentials: -- `.env` -- `.env.local` -- `.env.development.local` -- `.env.test.local` -- `.env.production.local` -- `.env*.local` (catch-all) +- [ ] 3.1 Check that `JWT_SECRET` in Vercel (Production, and Preview if legacy login is used there) is a long random value and not that old fallback. If you are not sure, set a new one and redeploy. Generate it with the command in `server/.env.example`: `node -e "console.log(require('crypto').randomBytes(64).toString('hex'))"`. done on: ____ -## Production Deployment +Effect of changing it: every legacy token (issued by `/api/login` and kept in the browser's `localStorage` under `token`) stops working, and those users must sign in again. Firebase sessions are not affected. Do not delete the variable: with it unset, `/api/login` returns 500 and legacy bearer tokens are rejected, which locks out legacy username/password users. -For Vercel deployments, environment variables should be set in the Vercel dashboard, not in committed files. The `.env.production` file serves only as a template showing which variables are needed. +### 4. Anything else + +- [ ] 4.1 `VITE_GEMINI_API_KEY`, `GEMINI_API_KEY` and `VITE_OCR_ENDPOINT` are documented in `docs/ENVIRONMENT_VARIABLES.md` and `app/.env.example`, but no code reads them, and no Google API key appears in history. If you ever created such keys, check Vercel and your machines, revoke them in the Google Cloud console and delete the variables. Anything prefixed `VITE_` is bundled into the browser. done on: ____ +- [ ] 4.2 Optional: restrict the Firebase web API key by HTTP referrer and API in the Google Cloud console, and consider Firebase App Check. The key is public by design, so this is hardening and not rotation. done on: ____ + +### 5. Close out + +- [ ] 5.1 Decide what to do about git history (see "Git history" below) and record it here: accepted the risk after rotation, or scrubbed it. Which, and why: ____ done on: ____ +- [ ] 5.2 Change the status block at the top of this file to say what was rotated and when, using the dates above, and to state the history decision from 5.1. done on: ____ +- [ ] 5.3 Optional, after rotation: trim the parts of `AUDIT_REPORT.md` that spell out how to pull the values out of history (the `git log -p` command and commit range). Not needed if you scrubbed history. done on: ____ +- [ ] 5.4 Close issue #22 with the dates from steps 1.7 and 2.1 and the history decision from 5.1. done on: ____ + +## Git history + +Rotating makes the leaked values worthless. The old commits stay readable either way, so you must choose what to do about them. This is your decision; nothing here does it automatically. + +- **Accept the risk once rotated (default).** After steps 1 and 2 the exposed Neon password and Stack key are dead, so history holds no working credential. No force-push, no coordination. Recommended unless you have a reason to scrub. +- **Scrub history.** Use `git filter-repo` (or BFG Repo-Cleaner; the Git project itself discourages `git filter-branch`), rewrite every branch and tag, and force-push all of them. Every collaborator must re-clone, and open pull requests will break. Even then, GitHub keeps `refs/pull/*` (read-only) and can serve commits by SHA, so you would also have to ask GitHub Support to remove cached views and unreachable commits, and removal is not guaranteed. Scrubbing without rotating first protects nothing. + +Whichever you choose, write it down in step 5.1 ("accepted the risk" or "scrubbed", with the date). Issue #22 asks for that decision to be on record. + +To see exactly which commits carry which rule hits, run this on a **full** copy (a shallow clone is partial; the tool warns when it detects one): + +```bash +git clone --mirror https://github.com/paccloud/Fish_Cost_Calculator.git fish-mirror.git +cd fish-mirror.git +node /path/to/your/checkout/scripts/check-secrets.mjs --history +``` + +It prints commit, file path, rule and a count, never the value. Hits are expected until history is scrubbed, which is why CI does not run this mode. It reads every line each commit added on every ref, including UTF-16 files and lines a merge conflict resolution introduced, together with unchanged context so that a value added under an unchanged `name:` line is still recognised (a lockfile keeps two lines, and the wider window only where a `-----BEGIN` line makes a PEM key block possible). A merge commit's result is also scanned when it differs from every parent (see the range scan below). Only matches that touch an added line are reported, so a later commit is not blamed for an old value that merely sits next to its change. If `git log` itself fails it exits 2 and prints git's first error line; treat that as "not scanned", never as "clean". + +## Prevention + +- **CI secret scan.** The `Secret scan` job in `.github/workflows/ci.yml` runs `node scripts/check-secrets.mjs` (Node 20, no dependencies, 5-minute timeout) on every pull request targeting `main` and every push to `main`. It scans every git-tracked file and fails the build on a hit. Run the same command locally before you push. The job then runs a second step, the commit-range scan (next bullet). + - **Commit-range scan in CI.** The tip scan only sees the checked-out files, so a value that one commit adds and a later commit removes would pass while it stays in the pushed history. The job therefore checks out full history (`fetch-depth: 0`, `persist-credentials: false`) and runs `node scripts/check-secrets.mjs --range ..`, which reads the ADDED lines of every commit reachable from `` but not from `` with the same reader as `--history` (added-line logic, the two-line context for split name/value pairs, the lockfile rules, combined diffs for merge commits, plus a scan of each merge's result for every file that differs from all of its parents, so a merge that completes a credential neither parent held, such as a name from one parent and its value from the other or the two halves of a PEM block, is reported on the merge, while a credential a parent already had is not blamed on it). The report prints the commit's short id, path, rule and a count, never matched text. Exit 1 on a hit; exit 2 (fail closed) when the range could not be read completely. The range cost is the size of the range, not of the repository (`git log --not `). Events: + - `pull_request`: `base.sha..head.sha`. A base that is not an ancestor of the head (the base branch moved on) is fine: the range is what the head has and the base does not, so commits that only exist on the base side are not blamed. A merge of `main` into the branch pulls the merged `main` commits into the range too; they were scanned when they landed, so this only costs a little time. A PR from a fork is scanned the same way: the checkout is the PR merge commit, whose second parent is the fork's head commit, so `head.sha` is present after the full fetch. The workflow uses `pull_request`, never `pull_request_target`, so fork code never runs with write access or secrets. + - `push` to `main`: `before..sha`. An all-zero `before` (the branch was just created) has no known base: only the pushed tip commit is scanned, and the output says so. A `before` that is no longer in the clone (a force-push dropped it, or a partial fetch) exits 2 with a message; it is never treated as clean and the job never falls back to the full-history audit. + - `workflow_dispatch`: no commit range exists, so only the tracked-file scan runs (the step says so). + - Edge cases: a range without commits passes (`no commits ... nothing to scan`); a shallow clone, an unresolvable base or head, an added file version over the size limit, or a range spec that is not `..` (a three-dot range is rejected) exits 2. The event values reach the shell only through environment variables, are checked to be plain commit ids and are quoted; nothing from the event is interpolated into a script. `permissions` stays `contents: read`. + - Limits, stated plainly: the scan cannot see a value that was never in a commit (a value pasted into a PR description, an issue, a comment or a CI log, or one held only in a local branch). A force-push that rewrites the branch can remove the evidence before the job runs, or between two runs; the scan only proves the commits that were pushed and still reachable when it ran, so a secret that was pushed and then erased by a force-push before the job ran is not seen. A leak that reached GitHub is still a leak even when the commit is gone from the branch, because the old commit stays fetchable: rotate it. The scanner runs from the PR's own checkout, so a PR that edits `scripts/check-secrets.mjs` or the workflow is judged by its own edited version: review changes to those two files by hand. It is a heuristic (see below), not a replacement for GitHub's secret scanning and push protection. + - What it scans: file contents decide, not names. The only deliberate exclusions are submodule pointers (no content in this repo) and verified-binary files (see below); `.claude/skills/`, other `*.lock` files and text files with an image-like extension are scanned. Lockfiles (exact names: `package-lock.json`, `npm-shrinkwrap.json`, `yarn.lock` (v1 and berry), `pnpm-lock.yaml`, `Cargo.lock`, `Gemfile.lock`, `poetry.lock`, `composer.lock`, `Pipfile.lock`, `bun.lock`, `bun.lockb`, `go.sum`) are NOT skipped any more: ordinary integrity digests (`sha512-...`, `sha1-...`, `h1:...`, hex checksums after a `checksum`/`shasum`/`hash` key or `#` commit fragment) are blanked, and then only targeted rules run on them, because the generic entropy and secret-name rules would report every hash. Those are: a password in any URL (`https://user:@registry/...`, in `resolved`, `tarball`, `url`, `source`, `remote`, `registry` or anywhere), a token as the URL user name, a credential-named query parameter (`token`, `sig`, `key`, `password`, `_authToken`, ...), an auth field (`_authToken`, `_auth`, `_password`, `npmAuthToken`, `npmAuthIdent`, `password`, `token`, `secret`, `client_secret`, `api_key`, ...; a dependency specifier such as `^1.0.0`, `npm:x`, `workspace:*` or `latest`, a placeholder and a `${...}` reference do not count), every provider-shaped token below, webhook URLs and PEM private keys. `--history` and `--range` apply the same lockfile rules to added lines. A lockfile may be up to 16 MB (other files 5 MB); a larger one fails the run like any oversize file. UTF-16 and UTF-32 files (BOM or not) are decoded, and a tracked symlink is scanned as its target text. File names are read as raw bytes, so a name that is not valid UTF-8 is still opened and scanned. The summary line says how many files were skipped and why. It fails closed: a tracked text file over 5 MB, or a tracked file that exists but cannot be read (permissions, a directory in its place, an I/O error), fails the build with a message naming the count and the paths. Both versions of a tracked file are scanned whenever they differ: the working-tree file AND the blob staged in the git index (compared by git object id, so a staged real value cannot be hidden by swapping in a placeholder before the commit; an unmerged path is scanned in every stage). When the two differ, the report says which version each finding is in (`(index)` or `(working tree)`, with path, line and rule only) and the summary counts the files that differ; an index blob over the size limit fails the run like any other oversize file. The staged blob is compared before any size or binary shortcut, so replacing a staged text file with a large binary in the working tree does not hide it, and every oversize report names the limit that applied to that path (`MAX_FILE_BYTES`, or `MAX_LOCKFILE_BYTES` for a lockfile). `--history` and `--range` read as much unchanged context around each change as the name/value pair matcher looks across, and blame the commit that added the value line. A tracked file that is missing from the working tree (deleted before commit, or sparse checkout) is scanned from the index alone, and the summary says so. In a CI checkout the two versions are identical, so this costs one hash per file and prints nothing extra. + - Rules: URLs with an embedded real password (any scheme, plus a password after `curl -u` / `--user` / `--proxy-user` / `-U` (the whole argument is judged, so a quoted `user:several words` cannot hide behind a placeholder-like first word) and after the password flags of `wget`, `mysql`, `mongosh`, `redis-cli`, `http`/`xh`, `sshpass`, `smbclient` and `ldapsearch`), PEM private keys, AWS access key IDs, Google API keys, GitHub tokens (`ghp_`, `gho_`, `ghu_`, `ghs_`, `ghr_`, `github_pat_`), the whole Slack token family (`xoxb-`, `xoxp-`, `xoxa-2`, `xoxr-`, `xoxs-`, `xoxo-` in its documented digit-group shape, app-level `xapp-`, configuration and refresh tokens `xoxe-` and `xoxe.xoxp-`; a hyphenated prose slug such as `xoxo-love-2026` and a repeated-character body pass), Stripe live keys and webhook secrets (`whsec_`), GitLab (`glpat-`, `gldt-`, `glrt-`, `glptt-`, ...), npm (`npm_`), PyPI, Twilio API key SIDs (`SK` + 32 hex; an account SID `AC...` is an identifier, not a credential, and is not reported), SendGrid (`SG.`), Mailgun (`key-`), Shopify (`shpat_`, `shpca_`, `shppa_`, `shpss_`), DigitalOcean (`dop_v1_`, `doo_v1_`, `dor_v1_`), Hugging Face (`hf_`), OpenAI (`sk-proj-`, `sk-svcacct-`, `sk-admin-` and legacy `sk-` keys), Anthropic (`sk-ant-`), Google OAuth (`ya29.` access tokens, `GOCSPX-` client secrets), Azure storage, Service Bus and Event Hubs keys in a connection string and SAS URL `sig=` signatures, Heroku (`HRKU-`, and `HEROKU_API_KEY` set to a UUID), Datadog API and application keys on a `DD_API_KEY` style name, Sentry (`sntrys_`, `sntryu_`, and a DSN that still carries its deprecated secret half `key:secret@`; the public key of a modern DSN is safe to ship in a browser bundle by design and passes), Doppler (`dp.st.` and the other `dp.` kinds), Vault (`hvs.`, `hvb.`, `hvr.`), Linear (`lin_api_`), Notion (`ntn_` and legacy `secret_`), Atlassian (`ATATT`) and Bitbucket app passwords (`ATBB`), Telegram bot tokens, Mapbox secret tokens (`sk.eyJ`), Square (`sq0atp-`, `sq0csp-`), Firebase FCM legacy server keys, New Relic (`NRAK-`, `NRII-`), Cloudflare (`cfut_`, `cfat_`, `cfk_`), Discord bot tokens, Databricks, Grafana service accounts, Supabase, PlanetScale, Docker Hub PATs, RubyGems, Terraform Cloud and age secret keys (each recognised by its fixed prefix and length wherever it appears, under any name and in any scanned file type, so a token in JSON, Markdown or source under a non-secret name is found; documentation placeholders such as `xxxx`, `<...>`, `your_...` and runs of one repeated character pass, and prose that only mentions a prefix passes), Neon API keys and role passwords (`napi_`, `npg_` prefixes), Stack Auth secret keys (`ssk_`), JWT-shaped tokens, SQL statements that set a role or user password to a literal (single, double, backtick, `$$dollar$$`, `N'...'` and `E'...'` quoting; an unquoted Oracle password after `IDENTIFIED BY` inside a CREATE, ALTER or GRANT statement; MySQL `IDENTIFIED WITH ... BY|AS` and `SET PASSWORD FOR ... = PASSWORD(...)`; SQL Server `CREATE LOGIN ... WITH PASSWORD`; MongoDB `createUser` with a `pwd` field; bind parameters and placeholders pass, and a stored hash is judged like any other literal), `jwt.sign`/`jwt.verify` called with a random-looking string literal, secret-like names assigned a literal (a quoted value counts as a whole even when it contains spaces, and an unquoted YAML, ini or dotenv value runs to the end of the line, but a quoted or rest-of-line value with spaces is exempt only as text: in a message catalog (a path under `i18n`, `locales`, `lang`, `messages`, `translations`, or a file such as `en.json`, `messages.yml`, `messages_de.properties`) when the whole value reads as a sentence (plain words, no letter-digit-symbol piece such as `Passw0rd!`, a bare number counts against it unless it counts something as in "at least 8 characters", at least half of the words common sentence vocabulary, or two thirds when it is not written like a sentence), and anywhere else only when it reads as documentation about the credential (three or more plain words with an instruction or reference cue such as "see", "ask", "step", "setup" or "the password"/"a token", e.g. "the password you chose during setup" or "set via environment variable at runtime"). `correct horse battery and staple`, `random long password 123!`, `the horse is 123` and a sentence of function words such as `This is the way.` or `My voice is my password.` are findings outside a message catalog; the tradeoff is that a UI sentence in an ordinary config file needs a placeholder value or the allow marker, and an ASCII sentence in another language is reported even in a catalog; YAML block scalars are read, also as the value of a split name/value pair (`- name: X` then `value: |` or `>-`, in either order, Helm values, Compose, GitHub Actions and CloudFormation `Value: !Sub |`), as are multi-line quoted values and JSON strings with `\n` escapes; also name/value pairs split across fields in any order, with other fields in between, in a bounded window (JSON and JSON stored as an escaped string, YAML flow mappings `{name: X, value: Y}`, YAML block mappings and Kubernetes or Compose env lists, Netlify-style nested values, Terraform blocks: the secret-like `name`/`key` field and a `value` field are matched inside the same `{...}` object (braces inside strings do not count), the same call `(...)`, the same XML entry (``, ``, ``), or the same YAML item, about 1,500 characters or up to 12 lines each way, so a value in a neighbouring object is not attributed to it and `valueFrom` references are not values; this applies to source code too when the name is a quoted literal, and covers CloudFormation `ParameterKey`/`ParameterValue`, Elastic Beanstalk `OptionName`, the value fields `value`, `val`, `content`, `data`, `default`, `defaultValue`, `stringValue`, `secretValue`, YAML tags and anchors (`!!str V`, `&a V`), a secret-like key with a nested `value` child (`NAME:` then `value: V`, `{"NAME":{"value":"V"}}`), and CSV, TSV and Markdown table rows (`NAME,V`, `| NAME | V |`); a file with so many candidate pairs that the search budget runs out is reported once as unverifiable), secrets on a command line (`secret-cli-command`: `gh secret set NAME --body V`, `vercel env add NAME <<< V`, a heredoc (the whole body is judged, so a passphrase is found as well as a token, and a body whose delimiter is not found within 200 lines is reported), or `echo V | vercel env add NAME` (the `echo`/`printf` may sit anywhere before the pipe: indented, in a YAML `run:` block, after `&&`, `then`, `sudo`, or on a continued line), `netlify env:set NAME V`, `aws ssm put-parameter --name NAME --value V`, `kubectl create secret generic --from-literal`, `az keyvault secret set`, `heroku config:set`, ...; environment references and placeholders pass); in env, shell and Compose files the literal default or alternate of a `${NAME:-literal}` expansion is checked, as is literal text next to a reference such as `${NAME}suffix`, while a pure `${NAME}`, `$NAME` or `${NAME:?message}` is not; the same expansion check applies to a password inside a URL or after `curl -u`, so a URL password of the form `${DB_PASSWORD:-}` is a finding while `user:${DB_PASSWORD}@host` is not; a name written as a quoted property, `config['JWT_SECRET'] = '...'` (single, double or backtick quotes, `obj?.[...]`, nested chains, `{ ['NAME']: '...' }`), is judged like `config.JWT_SECRET`; every assignment operator (`||=`, `&&=`, `??=`, `:=`, `?=`, `+=`, `-=`, `.=`, `<-`, `=>`, `->`, Kotlin `to`), typed declarations (Rust `const NAME: &str =`, Go `const NAME string =`, Nim `NAME* =`) and Dockerfile `ENV NAME value` are recognised), a URL held by a strong secret name when part of it is itself a credential (a signed-URL parameter such as `sig`, `X-Amz-Signature`, `token`, `access_token`, `key` or `password`, a random query value or path segment, a token used as the URL user name; an endpoint held by a weak name such as `TOKEN_ENDPOINT` or `TOKEN_URL`, and a URL with nothing credential-like in it, pass), webhook URLs whose path or query is the token in any file (`webhook-url`: Slack `hooks.slack.com/services|workflows|triggers/...`, Discord, Microsoft Teams and Power Automate, Zapier, IFTTT, PagerDuty integration and Telegram bot URLs; documentation placeholders such as `.../XXXXXXXX` or `` pass), native credential-file formats, matched by file name (variants such as `netrc.txt`, `.netrc.bak`, `dot-netrc`, `pgpass.local` count) and by content in any file, including Markdown notes and scripts that write them with `echo`, `printf` or a heredoc (`machine H login U password P`, five-field `.pgpass` lines with a numeric port, Docker `"auths"` JSON or YAML with a non-placeholder `auth`, `identitytoken` or `registrytoken`) (`credential-file`, plus stricter name/value handling with a 4-character minimum in `secret-assignment`: `.netrc`/`_netrc` `password`, `account` and multi-line `machine` blocks; `.pgpass` `host:port:db:user:password`; `.git-credentials` URL lines including a token used as the user name; `.npmrc`/`.yarnrc` `_authToken`, `_auth`, `_password`, scoped `//registry/:_authToken` keys and the quoted yarn form, `.yarnrc.yml` `npmAuthToken`/`npmAuthIdent`; `.pypirc` `password`; `.aws/credentials` `aws_secret_access_key` and `aws_session_token`; Docker `config.json`/`.dockercfg` `auth`, `identitytoken`, `registrytoken`; Kubernetes `.kube/config` and kubeconfig `token`, `password`, `client-key-data` (also in any YAML) and `.dockerconfigjson`; `.htpasswd`/`.htdigest` (a committed password hash counts as a credential because it can be cracked offline; a placeholder hash body such as `$apr1$xxxxxxxx$xxxxxxxx` passes, any other fixture needs the allow marker); the anonymous-FTP convention `password guest@` is not a secret; `.my.cnf` `password`; `.s3cfg`/`.boto` `secret_key`, `access_token`; Terraform `.terraformrc`, `*.tfrc.json`, tfvars and tfstate; `.curlrc` `user = "name:password"`; `.wgetrc`; `.vault-token`; placeholders, `${...}` and `<...>` references still pass), and `process.env. || "literal"` fallbacks (dot or bracket access, destructuring defaults, wrapped lines, Python `os.getenv`). A name may be written in any assignment shape, including inside another object literal or a minified one-line JSON (`x=1;NAME='v'`, `{"a":{"NAME":"v"}}`): a non-secret assignment never hides the one after it. + - Also covered: written-out HTTP credentials (`http-auth-credential`: `Authorization` / `Proxy-Authorization` with a `Bearer`, `Basic` (decoded, so `user:pass` and the RFC sample pass), `token`, `ApiKey` or `Digest` value in header text, `curl -H`, `headers.set(...)`, object, JSON and YAML fields and nginx, Apache and HAProxy header directives, plus Basic-auth calls with a literal password such as `auth=("u", "pw")` and `HTTPBasicAuth`, where the second argument is read whole in any quote style, so a quoted passphrase with spaces is judged like other quoted values); Terraform `variable` / `output` blocks whose secret-like label has a literal `default` or `value`; `pulumi config set` and `Pulumi..yaml`; and whitespace-delimited service settings (`config-directive-secret`: Redis `requirepass`, `masterauth`, `sentinel auth-pass`, ACL `>password` and `#sha256`, HAProxy userlists, Mosquitto, msmtp, fetchmail, nginx and Apache `SetEnv` / header / `fastcgi_param` directives, `ssl_passphrase_command`, OpenVPN inline credentials, `mosquitto_passwd -b`, `htpasswd -b`, `rabbitmqctl add_user`, `docker login -p`). References, placeholders, ansible-vault ciphertext, path-valued directives and `sensitive = true` alone pass. + - Also covered (round 15): Java properties files (`*.properties`, and with an environment or backup suffix such as `.properties.local`) with a blank as the separator (`key value`, `key\ with\ escaped\ space value`, `\` continuation lines), judged like `key=value` (message catalogs keep their prose exemption); function-call environment setters with a literal value (`Environment.SetEnvironmentVariable`, `os.Setenv`, `os.putenv`, `System.setProperty`, `.setProperty`, `getenv().put`, `ENV.store`, `Deno.env.set`, `env::set_var`, `setenv` / `putenv`, `SetEnvironmentVariableW`, Perl `$ENV{NAME} = 'v'`); and whole-value variable references that pass (`$name`, `$dbPassword`, `%(name)s`, `$ENV{NAME}`, `$_ENV['NAME']`, `$env:NAME`, `<%= ... %>`, `@name@`), while a literal glued to a reference, a `${NAME:-literal}` default and a password that contains `$` are still findings. + - Lockfiles and XML: lockfiles (`package-lock.json`, `yarn.lock`, `pnpm-lock.yaml`, `Cargo.lock`, ...) are scanned with targeted rules only (URL passwords, credential-named URL parameters, `_authToken`/`password`/`token` style fields, provider tokens, private keys), after ordinary integrity digests are blanked. A field value is skipped only when the whole value is a version or dependency specifier (`^1.2.3`, `>=1 <2`, `*`, `npm:x@1`, `workspace:*`, `latest`); a value that merely starts with a digit, `v` or `x` is judged as a credential. XML configuration files (`.xml`, `.config`, every `*proj` MSBuild file, `.props`, `.targets`, `.plist`, `.mobileconfig`, `.resx`, `.wsdl`, `.cscfg`, `.csdef`, `.jmx`, `.iml`, `.pom`, ... and any of them with a `.template`, `.dist`, `.sample`, `.example`, `.erb`, `.j2`, `.tpl` or `.bak` suffix) get configuration-value semantics, including namespace-prefixed elements (``), CDATA text and entries named by a `name=`/`key=` attribute; markup and schema formats (`.svg`, `.html`, `.xhtml`, `.xsl`, `.xaml`, `.xsd`, `.rss`, `.kml`) stay in code mode on purpose. Android `@string/...` references, `@token@`, `#{token}` and `%%TOKEN%%` substitutions, file-system paths and Maven's documented example values pass. + - How strict: a secret-like name is one that ends in `secret`, `password`/`pass`/`pwd`, `token` or a qualified `key` (`api_key`, `private_key`, `signing_key`, `encryption_key`, ...). In env, config and Markdown files such a name with any real-looking value of 8 or more characters is a finding, with no entropy test. In source code (JS, TS, Python, SQL, HTML, ...) only a quoted, random-looking literal counts, so ordinary identifiers and test fixtures do not trip it. XML configuration files get the configuration rules, not the source-code ones: `.xml` (Maven `settings.xml` and `pom.xml`, Ant, Tomcat `server.xml`, Android `strings.xml`), `.config` (`web.config`, `app.config`), MSBuild (`.csproj`, `.vbproj`, `.fsproj`, `.props`, `.targets`), `.plist`, `.resx`, `.wsdl`, `.nuspec`, `.pubxml` and `.settings`. So a `` whose `` is a secret-like name and whose `` is a passphrase with spaces is a finding in `settings.xml` exactly as it is in a `.config` file, and so is an element named like a secret (`...`, ``, ``) or a secret-named attribute. A Maven-encrypted password (`{base64}`), a `${env.NAME}` reference and a placeholder pass. `.svg`, `.html`, `.htm` and `.xhtml` stay in source-code mode: they are markup, so a form field named `password` or an SVG label is not a finding. + - Placeholders pass: `your_...`, `change-me`, `user:password@host`, `example`, `xxxx`, `<...>`, `REDACTED`, empty values, `process.env` / `import.meta.env` references, encrypted (`ENC[...]`) and hashed values, and a value cut off with a trailing `...`. A marker inside a longer value (`...`, `***`, ``) only counts when it stands for most of the value. Hosts `example.com`/`.org`/`.net`, `*.example`, `*.test`, `*.invalid` and loopback may sit next to a password; a host that merely contains the word "example" may not. One documentation sample is allowed in `docs/API.md` only, and only when it is the whole assigned value (the sample with anything added before or after it is still a finding). + - The report prints file, line and rule name only, never the matched text. + - A false positive: use an obvious placeholder, or put `check-secrets:allow` in a comment on that line (for a match that spans lines, any of its lines; for a name/value pair split across lines, the name line or the value line, in either order, but not an unrelated line in between). A real hit: remove the value **and rotate the credential**; deleting the line does not un-leak it. + - It is a heuristic over the checked-out files. It does not find a secret stored under a name that does not mention one (for example `DB_LOGIN=`), values built up in code, secrets inside binary or compressed files, or a raw `/` in a URL password. It scans what is tracked now (working tree and index), so a value that was committed and later removed is only seen through `--range` (the commits of one pull request or push, which CI runs) or `--history` (everything, owner-run). The credential-file, URL and JSON-pair checks recognise the formats listed above by file name and shape; a credential in some other tool's private format, or a pair whose name and value are further apart than the window, is not seen. A file counts as binary only with a NUL byte in its first 8 KB AND a known binary signature (PNG, JPEG, ZIP, PDF, ELF, gzip, ...); any other content, including a text file with a stray NUL byte, is decoded (NULs removed) and scanned. `--history` and `--range` apply the same test to the first 8 KB of each added file version, so a large binary asset (a PNG, a font) is skipped and counted (`skipped: binary`) instead of failing the run as oversize, while a text version over the size limit still exits 2. Text in an encoding other than UTF-8, UTF-16 or UTF-32 is read as UTF-8, which is enough for an ASCII secret. Limits that are bounds, not exclusions: an identifier longer than 1 KB is not treated as an assignment name, a quoted value longer than 4 KB and a YAML block scalar longer than 60 lines are only judged in part, a literal over several lines (heredoc, TOML or Python triple quotes, template literal, a quote closed on a later line, backslash or INI continuation) is read up to 200 lines and 32 KB and is reported as unverifiable when it does not end within that, and a URL password written as a command substitution with spaces (`$(echo ...)`) is not judged. It does not read history unless you pass `--history`, and it does not replace GitHub's own scanning. + - `--range ..` is the CI commit-range scan described above (also usable locally, for example `--range origin/main..HEAD` before you open a pull request). + - `--history` (owner-run) exits 1 on a hit and 2 when the audit is incomplete: a file version too large to scan, a shallow clone, or a git failure. In those cases it says how many versions were NOT scanned and never prints "no hits". Only commits reachable from a ref are read (not dangling objects or the reflog). + - Tests: `app/src/lib/__tests__/checkSecrets.test.js` (run by `cd app && npm test`). They include detection rates over generated secrets and scan-time limits on hostile input. +- [ ] **Owner: enable GitHub secret scanning, and confirm push protection stays on,** in the repository's Settings, under Code security. Only the owner can. Push protection (already working) blocks a push that contains a recognised provider token before it lands. done on: ____ +- **Tracked env files are a footgun.** `app/.env.development` and `app/.env.production` are tracked, so a real value pasted into them gets committed. Keep them placeholder-only (CI checks). `app/.env.example`, `docs/ENVIRONMENT_VARIABLES.md` and `CLAUDE.md` now tell developers to work in the untracked `app/.env.development.local`. Untracking the two files in favour of `app/.env.example` is worth considering, but it is your call: `scripts/` reads `app/.env.development`, and Vite loads `app/.env.production` during `vite build`. + +## For new developers + +Real values go only in untracked files. Never commit them, and never ask for credentials in chat, issues or PRs. + +1. Frontend (Vite): `cp app/.env.example app/.env.development.local`. Vite loads that file after `app/.env.development` and it overrides that file; it is gitignored. The `VITE_FIREBASE_*` values come from Firebase Console, Project settings, your web app. They are public identifiers, not secrets. Never copy the template over the tracked `app/.env.development` or `app/.env.production`. +2. Local Express server: create `server/.env` (see `server/.env.example`). It needs `FIREBASE_PROJECT_ID` (the same Firebase project as the frontend), `ALLOWED_ORIGINS`, and `JWT_SECRET` for legacy login. +3. Vercel-based local testing (`vercel dev`): use a root `.env.local` created by `vercel env pull`. +4. Database credentials come from the Neon console or from `vercel env pull`, not from a team lead's message. The scripts in `scripts/` (`import-fish-data-to-neon.js`, `migrate-sqlite-to-neon.js`) load only the tracked `app/.env.development`, and `dotenv` does not override variables already set in your shell. So `export DATABASE_URL=...` in your shell for that session instead of writing a real string into the tracked file. +5. Auth is Firebase (email/password and Google). Do not configure Stack Auth; it is retired. + +### Gitignored files + +`.gitignore` covers `.env`, `.env.local`, `.env.development.local`, `.env.test.local`, `.env.production.local` and the catch-all `.env*.local`, plus `*.db`, `*.sqlite`, `uploads/` and `.vercel`. + +Tracked, placeholders only: `app/.env.development`, `app/.env.production`, `app/.env.example`, `server/.env.example`. + +## Production deployment + +Set environment variables in the Vercel dashboard (Production, Preview and Development), not in committed files. The secret-bearing ones are `DATABASE_URL` and `JWT_SECRET`; `FIREBASE_PROJECT_ID` and `ALLOWED_ORIGINS` are public configuration. Changes only reach new deployments, so redeploy. `app/.env.production` is a template, but Vite loads it during the build: a `VITE_*` variable missing from Vercel silently falls back to its placeholder and breaks auth without an error. Consider pointing Preview deployments at a separate Neon branch instead of the production data. diff --git a/app/.env.example b/app/.env.example index 91468b1..e7a1db0 100644 --- a/app/.env.example +++ b/app/.env.example @@ -1,5 +1,6 @@ # Environment Variables Template -# Copy this file to .env.development for local development +# Copy this file to .env.development.local (untracked) for local development. +# Never copy it over the tracked .env.development or .env.production; see SECURITY_NOTICE.md. # API Configuration # For local development, use: http://localhost:3000 diff --git a/app/src/lib/__tests__/checkSecrets.test.js b/app/src/lib/__tests__/checkSecrets.test.js new file mode 100644 index 0000000..af62ec1 --- /dev/null +++ b/app/src/lib/__tests__/checkSecrets.test.js @@ -0,0 +1,7200 @@ +/** + * Tests for scripts/check-secrets.mjs (the repository secret scanner). + * + * Every fake credential below is assembled at RUNTIME from pieces, so this file + * contains no secret-shaped literal and passes the scanner itself (one test + * asserts exactly that). Do not paste a real credential here, not even to + * "test with a real one". + * + * Test runner: Vitest (run via `cd app && npm test`) + */ + +import { describe, it, expect, afterEach } from 'vitest'; +import { spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import { + RULES, + credentialFormats, + decodeText, + fileMode, + formatHistoryReport, + formatOversizeReport, + formatReport, + formatUnreadableReport, + HISTORY_CONTEXT, + isBinaryContent, + isPlaceholder, + scanText, + secretNameKind, + isLockfile, + isXmlConfigPath, + sanitizeLockfile, +} from '../../../../scripts/check-secrets.mjs'; + +// Generous, but still meaningful, wall-clock limits. Hostile-input tests guard against catastrophic (quadratic or worse) +// backtracking, which takes minutes on these inputs; a slow CI runner is several times slower than a laptop, not +// hundreds of times. SLOW_TEST_MS is the Vitest timeout of every test that spawns git or node or builds a big fixture. +// The allow marker, built at runtime so that only the tests that mean to use it carry it. +const ALLOW_MARKER = ['check-secrets', ':allow'].join(''); +const HOSTILE_LIMIT_MS = 8000; +const SLOW_TEST_MS = 120_000; +const SLOW = { timeout: SLOW_TEST_MS }; + +const THIS_FILE = fileURLToPath(import.meta.url); +const REPO_ROOT = path.resolve(path.dirname(THIS_FILE), '../../../..'); +const SCANNER = path.join(REPO_ROOT, 'scripts', 'check-secrets.mjs'); +const THIS_FILE_REL = 'app/src/lib/__tests__/checkSecrets.test.js'; + +// A backtracking regex blocks the JS thread, and Vitest's own timeout cannot interrupt it: an in-process hostile-input test +// would hang the worker instead of failing. Hostile scans therefore run in a child process with a hard kill timeout (the +// test then fails cleanly with a null status), and the child reports the time of each case. +const CHILD_KILL_MS = 100_000; +const SCAN_MODULE_IMPORT = `import { scanText } from ${JSON.stringify(pathToFileURL(SCANNER).href)};`; +function timeInChild(body) { + const result = spawnSync(process.execPath, ['--input-type=module', '-e', `${SCAN_MODULE_IMPORT}\nconst timings = [];\n${body}\nconsole.log(JSON.stringify(timings));`], { + cwd: REPO_ROOT, + encoding: 'utf8', + timeout: CHILD_KILL_MS, + maxBuffer: 16 * 1024 * 1024, + }); + expect(result.error, 'the child was killed: a scan is backtracking').toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + return JSON.parse(result.stdout.trim()); +} + +// --------------------------------------------------------------------------- +// Runtime fixture builders +// --------------------------------------------------------------------------- + +const ALNUM = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; +const LOWER = 'abcdefghijklmnopqrstuvwxyz'; +const LOWER_ALNUM = 'abcdefghijklmnopqrstuvwxyz0123456789'; +const UPPER = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ'; +const UPPER_ALNUM = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; +const HEX = '0123456789abcdef'; +const DIGITS = '0123456789'; +const BASE64 = `${ALNUM}+/`; +const PNG_HEAD = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0d]); +const PASSWORD_MANAGER = `${ALNUM}!$*-_.~+=^`; + +/** Deterministic pseudo-random string so failures are reproducible. */ +function randomString(length, seed, alphabet = ALNUM) { + let state = seed; + let out = ''; + for (let i = 0; i < length; i += 1) { + state = (state * 1103515245 + 12345) % 2147483648; + out += alphabet[Math.floor(state / 65536) % alphabet.length]; + } + return out; +} + +const base64url = (value) => + btoa(JSON.stringify(value)).replace(/=+$/, '').replace(/\+/g, '-').replace(/\//g, '_'); + +const secretName = (...parts) => parts.join(''); + +// Provider token families, one entry per shape (review round 9). Each value is assembled at runtime from a prefix and +// generated characters, so this file holds no token-shaped literal. `rule` is the scanner rule that must report it. +const B64URL = `${ALNUM}_-`; +const PROVIDER_TOKENS = (() => { + const r = (n, seed, alphabet = ALNUM) => randomString(n, seed, alphabet); + const digits = (n, seed) => r(n, seed, DIGITS); + return [ + ['slack-token', 'Slack app-level (xapp-)', () => ['xapp', '-1-A', digits(10, 1001), '-', digits(13, 1002), '-', r(64, 1003, HEX)].join('')], + ['slack-token', 'Slack configuration (xoxe.xoxp-)', () => ['xoxe', '.xoxp-1-', r(120, 1004, `${ALNUM}-`)].join('')], + ['slack-token', 'Slack refresh (xoxe-)', () => ['xoxe', '-1-', r(100, 1005, `${ALNUM}-`)].join('')], + ['slack-token', 'Slack legacy workspace (xoxa-2)', () => ['xox', 'a-2-', digits(12, 1006), '-', r(24, 1007)].join('')], + ['slack-token', 'Slack legacy (xoxr-)', () => ['xox', 'r-', digits(12, 1008), '-', r(24, 1009)].join('')], + ['slack-token', 'Slack legacy (xoxo-)', () => ['xox', 'o-', digits(12, 1010), '-', digits(12, 1011), '-', digits(12, 1054), '-', r(32, 1055, HEX)].join('')], + ['gitlab-token', 'GitLab personal (glpat-)', () => ['gl', 'pat-', r(26, 1012, B64URL)].join('')], + ['gitlab-token', 'GitLab deploy (gldt-)', () => ['gl', 'dt-', r(26, 1013, B64URL)].join('')], + ['gitlab-token', 'GitLab runner (glrt-)', () => ['gl', 'rt-', r(26, 1014, B64URL)].join('')], + ['npm-token', 'npm (npm_)', () => ['npm', '_', r(36, 1015)].join('')], + ['pypi-token', 'PyPI', () => ['pypi', '-AgEIcHlwaS5vcmc', r(70, 1016, B64URL)].join('')], + ['stripe-webhook-secret', 'Stripe webhook secret', () => ['whsec', '_', r(32, 1017)].join('')], + ['twilio-api-key', 'Twilio API key SID', () => ['S', 'K', r(32, 1018, HEX)].join('')], + ['sendgrid-api-key', 'SendGrid', () => ['SG', '.', r(22, 1019, B64URL), '.', r(43, 1020, B64URL)].join('')], + ['mailgun-api-key', 'Mailgun', () => ['key', '-', r(32, 1021, HEX)].join('')], + ['shopify-token', 'Shopify admin (shpat_)', () => ['shp', 'at_', r(32, 1022, HEX)].join('')], + ['shopify-token', 'Shopify custom app (shpca_)', () => ['shp', 'ca_', r(32, 1023, HEX)].join('')], + ['shopify-token', 'Shopify partner (shppa_)', () => ['shp', 'pa_', r(32, 1024, HEX)].join('')], + ['digitalocean-token', 'DigitalOcean (dop_v1_)', () => ['do', 'p_v1_', r(64, 1025, HEX)].join('')], + ['huggingface-token', 'Hugging Face (hf_)', () => ['hf', '_', r(34, 1026)].join('')], + ['openai-api-key', 'OpenAI project (sk-proj-)', () => ['sk', '-proj-', r(60, 1027, B64URL)].join('')], + ['openai-api-key', 'OpenAI legacy (sk-)', () => ['sk', '-', r(48, 1028)].join('')], + ['anthropic-api-key', 'Anthropic (sk-ant-)', () => ['sk', '-ant-api03-', r(80, 1029, B64URL)].join('')], + ['google-oauth-secret', 'Google OAuth access token (ya29.)', () => ['ya', '29.', r(60, 1030, B64URL)].join('')], + ['google-oauth-secret', 'Google OAuth client secret (GOCSPX-)', () => ['GOC', 'SPX-', r(28, 1031, B64URL)].join('')], + ['azure-storage-key', 'Azure storage connection string', () => ['DefaultEndpointsProtocol=https;AccountName=acct;Account', 'Key=', r(86, 1032, BASE64), '==;EndpointSuffix=core.windows.net'].join(''), (t) => t.match(/Key=([^;]+)/)[1]], + ['azure-storage-key', 'Azure SAS URL', () => ['https://acct.blob.core.windows.net/c/b?sv=2022-11-02&ss=b&srt=sco&sp=rl&se=2030-01-01T00%3A00%3A00Z&s', 'ig=', r(43, 1033), '%3D'].join(''), (t) => t.match(/sig=(.+)$/)[1]], + ['heroku-api-key', 'Heroku authorization token (HRKU-)', () => ['HR', 'KU-', r(60, 1034, B64URL)].join('')], + ['datadog-api-key', 'Datadog API key on a DD_API_KEY name', () => ['DD_API', '_KEY: "', r(32, 1035, HEX), '"'].join(''), (t) => t.slice(-33, -1)], + ['sentry-token', 'Sentry org auth token (sntrys_)', () => ['sntr', 'ys_', r(60, 1036, B64URL)].join('')], + ['sentry-token', 'Sentry DSN with its secret half', () => ['https://', r(32, 1037, HEX), ':', r(32, 1056, HEX), '@o123.ingest.sen', 'try.io/456'].join(''), (t) => t.slice(41, 73)], + ['doppler-token', 'Doppler service token', () => ['dp', '.st.dev.', r(44, 1038)].join('')], + ['vault-token', 'Vault service token (hvs.)', () => ['hv', 's.', r(90, 1039, B64URL)].join('')], + ['linear-api-key', 'Linear (lin_api_)', () => ['lin', '_api_', r(40, 1040)].join('')], + ['notion-token', 'Notion (ntn_)', () => ['nt', 'n_', r(46, 1041)].join('')], + ['notion-token', 'Notion legacy (secret_)', () => ['sec', 'ret_', r(43, 1042)].join('')], + ['atlassian-token', 'Atlassian API token (ATATT3)', () => ['AT', 'ATT3', r(70, 1043, `${ALNUM}_=-`)].join('')], + ['telegram-bot-token', 'Telegram bot token', () => [digits(9, 1057), ':A', 'A', r(33, 1058, B64URL)].join('')], + ['mapbox-secret-token', 'Mapbox secret token', () => ['sk', '.ey', 'J', r(40, 1059, B64URL), '.', r(22, 1060, B64URL)].join('')], + ['square-token', 'Square access token (sq0atp-)', () => ['sq0', 'atp-', r(22, 1061, B64URL)].join('')], + ['square-token', 'Square OAuth secret (sq0csp-)', () => ['sq0', 'csp-', r(43, 1062, B64URL)].join('')], + ['firebase-fcm-server-key', 'Firebase FCM legacy server key', () => ['AAAA', r(7, 1063, B64URL), ':APA', '91b', r(140, 1064, B64URL)].join('')], + ['newrelic-key', 'New Relic user key (NRAK-)', () => ['NR', 'AK-', r(27, 1065, UPPER_ALNUM)].join('')], + ['newrelic-key', 'New Relic ingest key (NRII-)', () => ['NR', 'II-', r(32, 1066, B64URL)].join('')], + ['cloudflare-token', 'Cloudflare API token (cfut_)', () => ['cf', 'ut_', r(48, 1067)].join('')], + ['discord-bot-token', 'Discord bot token', () => ['M', r(24, 1068, B64URL), '.', r(6, 1069, B64URL), '.', r(30, 1070, B64URL), '7A'].join('')], + ['other-provider-token', 'Databricks (dapi)', () => ['da', 'pi', r(32, 1044, HEX)].join('')], + ['other-provider-token', 'Grafana service account (glsa_)', () => ['gl', 'sa_', r(32, 1045), '_', r(8, 1046, HEX)].join('')], + ['other-provider-token', 'Supabase (sbp_)', () => ['sb', 'p_', r(40, 1047, HEX)].join('')], + ['other-provider-token', 'PlanetScale', () => ['psc', 'ale_tkn_', r(40, 1048, B64URL)].join('')], + ['other-provider-token', 'Docker Hub PAT', () => ['dckr', '_pat_', r(30, 1049, B64URL)].join('')], + ['other-provider-token', 'RubyGems', () => ['ruby', 'gems_', r(48, 1050, HEX)].join('')], + ['other-provider-token', 'Terraform Cloud', () => [r(14, 1051), '.atlas', 'v1.', r(70, 1052, B64URL)].join('')], + ['other-provider-token', 'age secret key', () => ['AGE-SECRET', '-KEY-1', r(58, 1053, 'QPZRY9X8GF2TVDW0S3JN54KHCE6MUA7L')].join('')], + ]; +})(); + +// Each case: which rule must fire, in what kind of file, and the secret part +// that must never appear in a report. `text` is the full file content. +const SECRET_CASES = (() => { + const dbPassword = randomString(24, 11); + const keyBody = randomString(64, 12, BASE64); + const awsBody = randomString(16, 13, UPPER_ALNUM); + const googleBody = randomString(35, 14); + const ghBody = randomString(36, 15); + const patBody = randomString(60, 16); + const slackBody = randomString(24, 17); + const stripeBody = randomString(24, 18); + const jwtSignature = randomString(43, 19); + const envSecret = randomString(32, 20); + const yamlSecret = randomString(32, 21); + const jsonSecret = randomString(40, 22); + const fallbackSecret = randomString(14, 23); + const napiBody = randomString(60, 24, LOWER_ALNUM); + const npgBody = randomString(12, 25); + const sskBody = randomString(32, 26); + const sqlSecret = randomString(24, 27); + const codeSecret = randomString(28, 28); + const shortSecret = randomString(12, 29); + const passphrase = ['correct-horse-battery-staple-', 'xxxx'].join(''); + const netrcPassword = randomString(22, 91); + const hookToken = randomString(24, 92); + const signature = randomString(30, 93); + const pairValue = randomString(26, 94); + + const scheme = ['postgres', 'ql'].join(''); + const pemHeader = ['-----BEGIN ', 'PRIVATE KEY-----'].join(''); + const rsaHeader = ['-----BEGIN RSA ', 'PRIVATE KEY-----'].join(''); + const jwtHead = base64url({ alg: 'HS256', typ: 'JWT' }); + const jwtBody = base64url({ sub: 'user-1234', role: 'admin' }); + + return [ + { + name: 'database URL with a password', + rule: 'url-password', + path: 'docs/setup.md', + secret: dbPassword, + text: `Connect with ${scheme}://svc_owner:${dbPassword}@db-host.internal:5432/appdb?sslmode=require\n`, + }, + { + name: 'mongodb+srv URL with a password', + rule: 'url-password', + path: 'config/db.json', + secret: dbPassword, + text: `{"uri": "${['mongodb', '+srv'].join('')}://app:${dbPassword}@cluster0.internal/app"}\n`, + }, + { + name: 'SQL Server URL with a password', + rule: 'url-password', + path: '.env', + secret: dbPassword, + text: `DATABASE_URL=${['ms', 'sql'].join('')}://admin:${dbPassword}@db.internal.corp/db\n`, + }, + { + name: 'driver-suffixed SQLAlchemy URL', + rule: 'url-password', + path: '.env', + secret: dbPassword, + text: `DATABASE_URL=${scheme}+psycopg2://app:${dbPassword}@db.internal.corp/db\n`, + }, + { + name: 'neo4j+s URL', + rule: 'url-password', + path: 'notes.md', + secret: dbPassword, + text: `uri ${['neo4j', '+s'].join('')}://neo4j:${dbPassword}@graph.internal.corp:7687\n`, + }, + { + name: 'HTTPS basic-auth URL', + rule: 'url-password', + path: 'notes.md', + secret: dbPassword, + text: `git clone https://deploy:${dbPassword}@gitlab.internal.org/x.git\n`, + }, + { + name: 'database URL whose host merely contains "example"', + rule: 'url-password', + path: '.env', + secret: dbPassword, + text: `DATABASE_URL=${scheme}://admin:${dbPassword}@ep-example-app-123456.us-east-2.aws.neon.tech/db\n`, + }, + { + name: 'curl -u user:password', + rule: 'url-password', + path: 'docs/runbook.md', + secret: dbPassword, + text: `curl -u admin:${dbPassword} https://api.internal.corp/v1/x\n`, + }, + { + name: 'PEM private key on separate lines', + rule: 'private-key-block', + line: 1, // reported on the header line + path: 'deploy/key.txt', + secret: keyBody, + text: `${pemHeader}\n${keyBody}\n${['-----END ', 'PRIVATE KEY-----'].join('')}\n`, + }, + { + name: 'RSA private key', + rule: 'private-key-block', + line: 1, + path: 'deploy/id_rsa.txt', + secret: keyBody, + text: `${rsaHeader}\n${keyBody}\n`, + }, + { + name: 'PEM private key escaped inside JSON', + rule: 'private-key-block', + path: 'service-account.json', + secret: keyBody, + text: `{"private_key": "${pemHeader}\\n${keyBody}\\n"}\n`, + }, + { + name: 'AWS access key id', + rule: 'aws-access-key-id', + path: 'notes.md', + secret: awsBody, + text: `id: ${['AKIA', awsBody].join('')}\n`, + }, + { + name: 'Google API key', + rule: 'google-api-key', + path: 'src/config.js', + secret: googleBody, + text: `const key = "${['AIza', googleBody].join('')}";\n`, + }, + { + name: 'GitHub personal access token', + rule: 'github-token', + path: 'notes.md', + secret: ghBody, + text: `token ${['gh', 'p_', ghBody].join('')}\n`, + }, + { + name: 'GitHub fine-grained token', + rule: 'github-token', + path: 'notes.md', + secret: patBody, + text: `token ${['github', '_pat_', patBody].join('')}\n`, + }, + { + name: 'Slack token', + rule: 'slack-token', + path: 'notes.md', + secret: slackBody, + text: `hook ${['xox', 'b-', '123456789012-', slackBody].join('')}\n`, + }, + { + name: 'Stripe live secret key', + rule: 'stripe-live-key', + path: 'notes.md', + secret: stripeBody, + text: `key ${['sk_', 'live_', stripeBody].join('')}\n`, + }, + { + name: 'Stripe live restricted key', + rule: 'stripe-live-key', + path: 'notes.md', + secret: stripeBody, + text: `key ${['rk_', 'live_', stripeBody].join('')}\n`, + }, + { + name: 'Neon API key in code', + rule: 'neon-api-key', + path: 'src/neon.js', + secret: napiBody, + text: `const k='${['na', 'pi_', napiBody].join('')}'\n`, + }, + { + name: 'Neon role password', + rule: 'neon-role-password', + path: 'notes.md', + secret: npgBody, + text: `password ${['np', 'g_', npgBody].join('')}\n`, + }, + { + name: 'Stack Auth secret server key in code', + rule: 'stack-auth-secret-key', + path: 'src/stack.js', + secret: sskBody, + text: `export const stack = { ${secretName('secretServer', 'Key')}: '${['ss', 'k_', sskBody].join('')}' };\n`, + }, + { + name: 'JWT-shaped token', + rule: 'jwt-token', + path: 'notes.md', + secret: jwtSignature, + text: `Authorization: Bearer ${[jwtHead, jwtBody, jwtSignature].join('.')}\n`, + }, + { + name: 'SQL ALTER ROLE ... PASSWORD', + rule: 'sql-password-literal', + path: 'db/rotate.sql', + secret: sqlSecret, + text: `ALTER ROLE neondb_owner WITH PASSWORD '${sqlSecret}';\n`, + }, + { + name: 'SQL CREATE ROLE ... LOGIN PASSWORD in a runbook', + rule: 'sql-password-literal', + path: 'docs/runbook.md', + secret: sqlSecret, + text: `CREATE ROLE app LOGIN PASSWORD '${sqlSecret}';\n`, + }, + { + name: 'MySQL IDENTIFIED BY', + rule: 'sql-password-literal', + path: 'db/users.sql', + secret: sqlSecret, + text: `CREATE USER 'app'@'%' IDENTIFIED BY '${sqlSecret}';\n`, + }, + { + name: 'SECRET in an .env file', + rule: 'secret-assignment', + path: 'app/.env', + secret: envSecret, + text: `# comment\n${secretName('JWT_', 'SECRET')}=${envSecret}\n`, + }, + { + name: 'PASSWORD in an .env.production file', + rule: 'secret-assignment', + path: 'app/.env.production', + secret: envSecret, + text: `${secretName('DB_', 'PASSWORD')}=${envSecret}\n`, + }, + { + name: 'DB_PASS (abbreviated name)', + rule: 'secret-assignment', + path: '.env', + secret: envSecret, + text: `${secretName('DB_', 'PASS')}=${envSecret}\n`, + }, + { + name: 'SENDGRID_KEY (vendor key name)', + rule: 'secret-assignment', + path: '.env', + secret: envSecret, + text: `${secretName('SENDGRID_', 'KEY')}=${envSecret}\n`, + }, + { + name: 'SERVICE_ROLE_KEY', + rule: 'secret-assignment', + path: '.env', + secret: envSecret, + text: `${secretName('SUPABASE_SERVICE_ROLE_', 'KEY')}=${envSecret}\n`, + }, + { + name: 'ENCRYPTION_KEY (qualified key name)', + rule: 'secret-assignment', + path: '.env', + secret: envSecret, + text: `${secretName('ENCRYPTION_', 'KEY')}=${envSecret}\n`, + }, + { + name: 'export in .envrc', + rule: 'secret-assignment', + path: '.envrc', + secret: envSecret, + text: `export ${secretName('API_', 'TOKEN')}=${envSecret}\n`, + }, + { + name: 'env.production without a leading dot', + rule: 'secret-assignment', + path: 'env.production', + secret: envSecret, + text: `${secretName('DB_', 'PASSWORD')}=${envSecret}\n`, + }, + { + name: 'Makefile variable', + rule: 'secret-assignment', + path: 'Makefile', + secret: envSecret, + text: `${secretName('SIGNING_', 'KEY')} = ${envSecret}\n`, + }, + { + name: 'AWS credentials file', + rule: 'secret-assignment', + path: '.aws/credentials', + secret: envSecret, + text: `[default]\n${secretName('aws_secret_access_', 'key')} = ${envSecret}\n`, + }, + { + name: 'TOKEN with export and quotes', + rule: 'secret-assignment', + path: 'scripts/setup.sh', + secret: envSecret, + text: `export ${secretName('AUTH_', 'TOKEN')}="${envSecret}"\n`, + }, + { + name: 'PRIVATE_KEY in YAML', + rule: 'secret-assignment', + path: 'config/app.yml', + secret: yamlSecret, + text: `service:\n ${secretName('private_', 'key')}: ${yamlSecret}\n`, + }, + { + name: 'API_KEY in JSON (camelCase name)', + rule: 'secret-assignment', + path: 'config/app.json', + secret: jsonSecret, + text: `{\n "${secretName('api', 'Key')}": "${jsonSecret}"\n}\n`, + }, + { + name: 'SECRET in a Markdown code block', + rule: 'secret-assignment', + path: 'docs/setup.md', + secret: envSecret, + text: `\`\`\`\n${secretName('SESSION_', 'SECRET')}=${envSecret}\n\`\`\`\n`, + }, + { + name: 'password in Terraform', + rule: 'secret-assignment', + path: 'infra/main.tf', + secret: codeSecret, + text: `resource "db" "main" {\n ${secretName('pass', 'word')} = "${codeSecret}"\n}\n`, + }, + { + name: 'quoted UPPER_SNAKE constant in JavaScript', + rule: 'secret-assignment', + path: 'server/auth.js', + secret: codeSecret, + text: `const ${secretName('JWT_', 'SECRET')} = '${codeSecret}';\n`, + }, + { + name: 'password key of a connection pool object', + rule: 'secret-assignment', + path: 'api/db.js', + secret: codeSecret, + text: `const pool = new Pool({ host: 'h', user: 'neondb_owner', ${secretName('pass', 'word')}: '${codeSecret}' });\n`, + }, + { + name: 'exported constant in TypeScript', + rule: 'secret-assignment', + path: 'src/keys.ts', + secret: codeSecret, + text: `export const ${secretName('api', 'Key')}: string = "${codeSecret}";\n`, + }, + { + name: 'property in a JSX file', + rule: 'secret-assignment', + path: 'src/App.jsx', + secret: codeSecret, + text: `const config = { ${secretName('api', 'Key')}: "${codeSecret}" };\n`, + }, + { + name: 'property in a Vue single-file component', + rule: 'secret-assignment', + path: 'src/App.vue', + secret: codeSecret, + text: `\n`, + }, + { + name: 'inline script in an HTML page', + rule: 'secret-assignment', + path: 'public/index.html', + secret: codeSecret, + text: `\n`, + }, + { + name: 'password variable in Python', + rule: 'secret-assignment', + path: 'tools/db.py', + secret: codeSecret, + text: `${secretName('pass', 'word')} = "${codeSecret}"\n`, + }, + { + name: 'SESSION_TOKEN of only 12 characters', + rule: 'secret-assignment', + path: '.env', + secret: shortSecret, + text: `${secretName('SESSION_', 'TOKEN')}=${shortSecret}\n`, + }, + { + name: 'passphrase-style secret', + rule: 'secret-assignment', + path: '.env', + secret: passphrase, + text: `${secretName('JWT_', 'SECRET')}=${passphrase}\n`, + }, + { + name: 'value cut by ";" (first part too short to look random)', + rule: 'secret-assignment', + line: 1, + path: '.env', + secret: envSecret, + text: `${secretName('DB_', 'PASSWORD')}=${envSecret.slice(0, 3)};${envSecret.slice(3)}\n`, + }, + { + name: 'value cut by "#" (first part too short to look random)', + rule: 'secret-assignment', + line: 1, + path: '.env', + secret: envSecret, + text: `${secretName('DB_', 'PASSWORD')}=${envSecret.slice(0, 3)}#${envSecret.slice(3)}\n`, + }, + { + name: 'value cut by "," (first part too short to look random)', + rule: 'secret-assignment', + line: 1, + path: '.env', + secret: envSecret, + text: `${secretName('DB_', 'PASSWORD')}=${envSecret.slice(0, 10)},${envSecret.slice(10)}\n`, + }, + { + name: 'k8s name/value pair split over two lines', + rule: 'secret-name-value-pair', + line: 2, // reported on the "- name:" line; the value is on line 3 + path: 'k8s/deploy.yaml', + secret: envSecret, + text: `env:\n - name: ${secretName('DB_', 'PASSWORD')}\n value: "${envSecret}"\n`, + }, + { + name: 'Vercel-style {"key","value"} pair', + rule: 'secret-name-value-pair', + path: 'vercel-env.json', + secret: envSecret, + text: `[{"key":"${secretName('JWT_', 'SECRET')}","value":"${envSecret}"}]\n`, + }, + { + name: 'jwt.sign with a string literal key', + rule: 'hardcoded-signing-key', + path: 'api/login.js', + secret: codeSecret, + text: `const t = jwt.sign({ id: user.id }, '${codeSecret}', { expiresIn: '1h' });\n`, + }, + { + name: 'hardcoded fallback for a secret env var', + rule: 'hardcoded-secret-fallback', + path: 'server/server.js', + secret: fallbackSecret, + text: `const key = process.env.JWT_${'SECRET'} || '${fallbackSecret}';\n`, + }, + { + name: 'hardcoded fallback with ?? and double quotes', + rule: 'hardcoded-secret-fallback', + path: 'api/auth.mjs', + secret: fallbackSecret, + text: `export const key = process.env.SIGNING_${'TOKEN'} ?? "${fallbackSecret}";\n`, + }, + { + name: 'fallback through bracket access', + rule: 'hardcoded-secret-fallback', + path: 'server/server.js', + secret: fallbackSecret, + text: `const key = process.env['JWT_${'SECRET'}'] || '${fallbackSecret}';\n`, + }, + { + name: 'fallback as a destructuring default', + rule: 'hardcoded-secret-fallback', + path: 'server/server.js', + secret: fallbackSecret, + text: `const { JWT_${'SECRET'} = '${fallbackSecret}', PORT } = process.env;\n`, + }, + { + name: 'fallback with a lowercase variable name', + rule: 'hardcoded-secret-fallback', + path: 'server/server.js', + secret: fallbackSecret, + text: `const key = process.env.jwt_${'secret'} || '${fallbackSecret}';\n`, + }, + { + name: 'fallback for DB_PASS (name outside SECRET/PASSWORD/TOKEN)', + rule: 'hardcoded-secret-fallback', + path: 'server/db.js', + secret: fallbackSecret, + text: `const pass = process.env.DB_${'PASS'} || '${fallbackSecret}';\n`, + }, + { + name: 'fallback for ENCRYPTION_KEY', + rule: 'hardcoded-secret-fallback', + path: 'server/crypto.js', + secret: fallbackSecret, + text: `const key = process.env.ENCRYPTION_${'KEY'} || '${fallbackSecret}';\n`, + }, + { + name: 'fallback with the literal on the next line (Prettier wrapping)', + rule: 'hardcoded-secret-fallback', + line: 1, + path: 'server/server.js', + secret: fallbackSecret, + text: `const key = process.env.JWT_${'SECRET'} ||\n '${fallbackSecret}';\n`, + }, + { + name: 'Python getenv default', + rule: 'hardcoded-secret-fallback', + path: 'tools/db.py', + secret: fallbackSecret, + text: `key = os.getenv("JWT_${'SECRET'}", "${fallbackSecret}")\n`, + }, + { + name: '.netrc password', + rule: 'credential-file', + path: 'home/.netrc', + secret: netrcPassword, + text: `machine api.internal login app password ${netrcPassword}\n`, + }, + { + name: 'webhook URL with a path token', + rule: 'webhook-url', + path: 'docs/alerts.md', + secret: hookToken, + text: `curl -X POST https://hooks.slack.com/services/T0123ABCD/B0123ABCD/${hookToken}\n`, + }, + { + name: 'signed URL held by a secret name', + rule: 'secret-assignment', + path: '.env', + secret: signature, + text: `${['API_', 'TOKEN'].join('')}=https://download.internal.org/file?sig=${signature}\n`, + }, + { + name: 'JSON name and value in reverse order with a field in between', + rule: 'secret-name-value-pair', + path: 'exports/env.json', + line: 1, + secret: pairValue, + text: `[{"value":"${pairValue}","type":"encrypted","key":"${['JWT_', 'SECRET'].join('')}"}]\n`, + }, + { + name: 'secret set on a command line', + rule: 'secret-cli-command', + path: 'deploy.sh', + line: 1, + secret: pairValue, + text: `gh secret set ${['JWT_', 'SECRET'].join('')} --body ${pairValue}\n`, + }, + { + name: 'Authorization header with an opaque bearer token', + rule: 'http-auth-credential', + path: 'docs/api.md', + line: 1, + secret: pairValue, + text: `curl -H "${['Author', 'ization'].join('')}: Bearer ${pairValue}" https://api.internal.corp/v1/items\n`, + }, + { + name: 'requirepass in redis.conf', + rule: 'config-directive-secret', + path: 'deploy/redis.conf', + line: 1, + secret: pairValue, + text: `${['require', 'pass'].join('')} ${pairValue}\n`, + }, + { + name: 'token as the user name in a lockfile dependency URL', + rule: 'lockfile-credential', + path: 'package-lock.json', + secret: dbPassword, + text: `{"packages":{"node_modules/x":{"resolved":"https://${dbPassword}@registry.internal/x/-/x-1.0.0.tgz"}}}\n`, + }, + ...PROVIDER_TOKENS.map(([rule, label, build, secretPart], index) => { + const token = build(); + return { name: `provider token: ${label}`, rule, path: index % 2 ? 'notes/findings.md' : 'src/data.json', secret: secretPart ? secretPart(token) : token, text: `${index % 2 ? 'see ' : '{"note":"'}${token}${index % 2 ? '' : '"}'}\n` }; + }), + ]; +})(); + +function windows(secret, size = 8) { + const out = []; + for (let i = 0; i + size <= secret.length; i += 1) out.push(secret.slice(i, i + size)); + return out; +} + +// --------------------------------------------------------------------------- +// Rule coverage +// --------------------------------------------------------------------------- + +describe('fixtures', () => { + it('generated fake secrets do not look like placeholders', () => { + for (const c of SECRET_CASES) expect(isPlaceholder(c.secret), c.name).toBe(false); + }); + + it('every scanner rule has at least one positive case', () => { + const covered = new Set(SECRET_CASES.map((c) => c.rule)); + expect([...covered].sort()).toEqual(RULES.map((r) => r.id).sort()); + }); +}); + +describe('scanText: detection', () => { + for (const c of SECRET_CASES) { + it(`detects: ${c.name}`, () => { + const findings = scanText(c.path, c.text); + expect(findings.map((f) => f.rule)).toContain(c.rule); + const hit = findings.find((f) => f.rule === c.rule); + const expectedLine = c.line ?? c.text.split('\n').findIndex((l) => l.includes(c.secret)) + 1; + expect(hit.line).toBe(expectedLine); + expect(hit.path).toBe(c.path); + }); + } + + it('reports the correct line number in a longer file', () => { + const secret = randomString(32, 31); + const text = `A=1\nB=2\n\n${['API_', 'KEY'].join('')}=${secret}\nC=3\n`; + expect(scanText('.env', text)).toEqual([{ path: '.env', line: 4, rule: 'secret-assignment' }]); + }); + + it('handles CRLF line endings', () => { + const secret = randomString(32, 32); + const text = `A=1\r\n${['API_', 'KEY'].join('')}=${secret}\r\n`; + expect(scanText('.env', text).map((f) => f.line)).toEqual([2]); + }); + + it('in source code only a QUOTED literal counts; a bare KEY=value is an expression', () => { + const secret = randomString(32, 33); + const name = ['API_', 'KEY'].join(''); + expect(scanText('src/config.js', `${name}=${secret}\n`)).toEqual([]); + expect(scanText('src/config.js', `const ${name} = "${secret}";\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + expect(scanText('.env.local', `${name}=${secret}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + expect(scanText('production.env', `${name}=${secret}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + }); + + it('in env and config files any non-placeholder value of 8+ characters counts, with no entropy test', () => { + const name = ['API_', 'TOKEN'].join(''); + const hex = randomString(40, 35, HEX); + const camel = 'ThisIsAVeryLongCamelCaseIdentifierNameHere'; + const kebab = 'my-long-descriptive-config-name-value'; + for (const value of [hex, camel, kebab, randomString(8, 36), 'a'.repeat(9)]) { + expect(scanText('.env', `${name}=${value}\n`).map((f) => f.rule), value.length).toEqual(['secret-assignment']); + } + expect(scanText('.env', `${name}=${randomString(7, 37)}\n`)).toEqual([]); + }); + + it('a name that only MENTIONS a secret needs a random-looking value', () => { + const endpoint = ['TOKEN_', 'ENDPOINT'].join(''); + const salt = ['PASSWORD_', 'HINT'].join(''); + expect(scanText('.env', `${endpoint}=oauth-token-endpoint-name\n`)).toEqual([]); + expect(scanText('.env', `${salt}=${randomString(24, 38)}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + }); + + it('digit-free single-case random secrets are found (they used to be missed)', () => { + const name = ['SECRET_', 'SEED'].join(''); + expect(scanText('.env', `${name}=${randomString(32, 39, LOWER)}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + expect(scanText('.env', `${name}=${randomString(32, 40, UPPER)}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + }); + + it('honors the inline check-secrets:allow marker on that line only', () => { + const secret = randomString(32, 34); + const name = ['API_', 'KEY'].join(''); + const allowed = `${name}=${secret} # check-secrets:allow\n`; + expect(scanText('.env', allowed)).toEqual([]); + const second = `${allowed}${name}=${secret}\n`; + expect(scanText('.env', second).map((f) => f.line)).toEqual([2]); + }); + + it('honors the marker on any line a multi-line match spans', () => { + const secret = randomString(14, 41); + const text = `const key = process.env.JWT_${'SECRET'} ||\n '${secret}'; // check-secrets:allow\n`; + expect(scanText('server/server.js', text)).toEqual([]); + }); + + it('a minified single-line file with thousands of matches still reports one finding per rule and line', () => { + const name = ['API_', 'KEY'].join(''); + const secret = randomString(32, 42); + const text = Array.from({ length: 2000 }, () => `${name}="${secret}"`).join(' '); + expect(scanText('bundle.min.json', text)).toEqual([{ path: 'bundle.min.json', line: 1, rule: 'secret-assignment' }]); + }); +}); + +describe('secretNameKind', () => { + it.each([ + ['JWT_SECRET', 'strong'], + ['jwtSecret', 'strong'], + ['db.password', 'strong'], + ['DB_PASS', 'strong'], + ['DB_PWD', 'strong'], + ['MYSQL_PWD', 'strong'], + ['AUTH_TOKEN', 'strong'], + ['refresh_token2', 'strong'], + ['apiKey', 'strong'], + ['STRIPE_API_KEY', 'strong'], + ['ENCRYPTION_KEY', 'strong'], + ['SIGNING_KEY', 'strong'], + ['SESSION_KEY', 'strong'], + ['HMAC_KEY', 'strong'], + ['aws_secret_access_key', 'strong'], + ['PRIVATE_KEY', 'strong'], + ['SERVICE_ROLE_KEY', 'strong'], + ['SENDGRID_KEY', 'weak'], + ['STRIPE_KEY', 'weak'], + ['TOKEN_ENDPOINT', 'weak'], + ['PASSWORD_MIN_LENGTH', 'weak'], + ['API_KEY_HEADER', 'weak'], + ['CREDENTIALS', 'weak'], + ['SALT', 'weak'], + ['max_tokens', 'weak'], + ['tokenizer', 'weak'], + ['PORT', null], + ['bypass', null], + ['compass', null], + ['KEY', null], + ['keyboard', null], + ['keyword', null], + ['DATABASE_URL', null], + ])('%s is %s', (name, expected) => { + expect(secretNameKind(name)).toBe(expected); + }); +}); + +describe('fileMode', () => { + it.each([ + ['.env', 'config'], + ['app/.env.production', 'config'], + ['production.env', 'config'], + ['env.production', 'config'], + ['.envrc', 'config'], + ['Makefile', 'config'], + ['.aws/credentials', 'config'], + ['infra/main.tf', 'config'], + ['k8s/deploy.yaml', 'config'], + ['config.json', 'config'], + ['docs/setup.md', 'prose'], + ['notes.txt', 'prose'], + ['server/server.js', 'code'], + ['src/App.jsx', 'code'], + ['src/App.vue', 'code'], + ['tools/db.py', 'code'], + ['db/rotate.sql', 'code'], + ['public/index.html', 'code'], + ])('%s is %s', (file, mode) => { + expect(fileMode(file)).toBe(mode); + }); +}); + +// --------------------------------------------------------------------------- +// Measured detection rates. These are the shapes that used to slip through: the old gate missed +// 18-19% of 32-character hex secrets and 10-16% of 16-character alphanumeric ones in env files. +// --------------------------------------------------------------------------- + +describe('scanText: detection rate over generated secrets', () => { + const uuid = (seed) => + [randomString(8, seed, HEX), randomString(4, seed + 1, HEX), `4${randomString(3, seed + 2, HEX)}`, `a${randomString(3, seed + 3, HEX)}`, randomString(12, seed + 4, HEX)].join('-'); + + const GENERATORS = [ + ['alnum-12', (s) => randomString(12, s), 0.85], + ['alnum-16', (s) => randomString(16, s), 0.95], + ['alnum-20', (s) => randomString(20, s), 0.95], + ['alnum-24', (s) => randomString(24, s), 0.97], + ['alnum-32', (s) => randomString(32, s), 0.97], + ['hex-16', (s) => randomString(16, s, HEX), 0], + ['hex-32', (s) => randomString(32, s, HEX), 0.99], + ['hex-40', (s) => randomString(40, s, HEX), 0.99], + ['uuid-v4', uuid, 0.99], + ['base64-24', (s) => randomString(24, s, BASE64), 0.97], + ['base64-44', (s) => randomString(44, s, BASE64), 0.97], + ['lowercase-32', (s) => randomString(32, s, LOWER), 0.95], + ['uppercase-32', (s) => randomString(32, s, UPPER), 0.95], + ['digits-20', (s) => randomString(20, s, DIGITS), 0.99], + ['password-manager-20', (s) => randomString(20, s, PASSWORD_MANAGER), 0.95], + ['prefixed-16', (s) => `pk_${randomString(12, s)}`, 0.85], + ]; + const SAMPLES = 150; + const name = (...parts) => parts.join(''); + + describe.each(GENERATORS)('%s', (label, make, codeRate) => { + const values = Array.from({ length: SAMPLES }, (_, i) => make(1000 + i * 7)); + + it('every value is found as an env-file secret, quoted or not (no length or entropy test)', () => { + const shapes = [ + (v) => ['.env', `${name('JWT_', 'SECRET')}=${v}\n`], + (v) => ['app/.env.production', `${name('DB_', 'PASSWORD')}=${v}\n`], + (v) => ['.env', `${name('DB_', 'PASSWORD')}="${v}"\n`], + (v) => ['.env', `${name('JWT_', 'SECRET')}='${v}'\n`], + ]; + let misses = 0; + for (const v of values) { + for (const shape of shapes) { + const [file, text] = shape(v); + if (scanText(file, text).length === 0) misses += 1; + } + } + expect(misses).toBe(0); + }); + + it(`is found as a quoted literal in source code at least ${codeRate * 100}% of the time`, () => { + const shapes = [ + (v) => ['a.js', `const ${name('JWT_', 'SECRET')} = '${v}';\n`], + (v) => ['a.js', `new Pool({ ${name('pass', 'word')}: '${v}' });\n`], + (v) => ['a.py', `${name('pass', 'word')} = "${v}"\n`], + ]; + let total = 0; + let hits = 0; + for (const v of values) { + for (const shape of shapes) { + const [file, text] = shape(v); + total += 1; + if (scanText(file, text).length > 0) hits += 1; + } + } + expect(hits / total).toBeGreaterThanOrEqual(codeRate); + }); + }); +}); + +// --------------------------------------------------------------------------- +// Placeholders and references must not be flagged +// --------------------------------------------------------------------------- + +describe('scanText: placeholders and references', () => { + const dbScheme = ['postgres', 'ql'].join(''); + const realish = randomString(24, 41); + + const CLEAN_ENV_LINES = [ + `DATABASE_URL=${dbScheme}://user:password@host/dbname`, + `DATABASE_URL=${dbScheme}://USER:PASSWORD@HOST:5432/DB`, + `DATABASE_URL=${dbScheme}://user:[YOUR-PASSWORD]@host/db`, + `DATABASE_URL=${['mongodb', '+srv'].join('')}://:@cluster.example.net/db`, + `DATABASE_URL=${dbScheme}://app:\${DB_PASSWORD}@db/app`, + `DATABASE_URL=${dbScheme}://app:...@db/app`, + `DATABASE_URL=${dbScheme}://app:${realish}@db.example.com/app`, + `DATABASE_URL=${dbScheme}://app:${realish}@db.example.org:5432/app`, + `DATABASE_URL=${dbScheme}://app:${realish}@service.example/app`, + `DATABASE_URL=${dbScheme}://app:${realish}@db.internal.test/app`, + `DATABASE_URL=${dbScheme}://app:${realish}@localhost:5432/app`, + `DATABASE_URL=${dbScheme}://app:${realish}@127.0.0.1/app`, + 'JWT_SECRET=your_jwt_secret_here', + 'JWT_SECRET=change-me-to-a-long-random-string-0123456789', + 'JWT_SECRET=changeme', + 'SESSION_TOKEN=', + 'SESSION_TOKEN=""', + 'AUTH_TOKEN=', + 'STRIPE_API_KEY=xxxxxxxxxxxxxxxxxxxxxxxx', + 'OTHER_SECRET=REDACTED', + 'OTHER_PASSWORD=...', + 'OTHER_PASSWORD=************', + 'VITE_API_KEY=$VITE_API_KEY', + 'VITE_API_KEY=${VITE_API_KEY}', + 'API_TOKEN=example-token-1234567890abcdef', + 'SECRET_KEY=process.env.SECRET_KEY', + 'SECRET_KEY=import.meta.env.VITE_SECRET_KEY', + 'CLIENT_SECRET=this_is_a_dummy_value_9999', + 'WEBHOOK_TOKEN=fake_token_for_local_dev_1', + 'VITE_GEMINI_API_KEY=AIza...', + 'GITHUB_TOKEN=ghp_...', + 'AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY', + 'PASSWORD_HASH=$2b$10$abcdefghijklmnopqrstuuABCDEFGHIJKLMNOPQRSTUVWXYZ012345', + 'DB_PASSWORD=ENC[AES256_GCM,data:abcdefghijklmnop,type:str]', + 'ORDINARY_SETTING=abcdefghijklmnopqrstuvwxyz0123456789', + 'TOKEN_ENDPOINT=https://auth.internal.test/oauth/token', + 'PASSWORD_MIN_LENGTH=12', + 'API_KEY_HEADER=x-api-key', + 'PASSWORD_FILE=/run/secrets/db_password', + 'max_tokens=100000000', + 'SECRET_NAME=fish-db-credentials', + 'CACHE_KEY=fish-data-v3', + 'STORAGE_KEY=fishCalcState_v2', + 'PARTITION_KEY=user_id', + 'DB_PASSWORD=password', + 'DB_PASSWORD=required', + 'token=localStorage.getItem("token");', + 'token=req.body.token', + 'DB_PASSWORD=$(cat /run/secrets/db)', + 'docker run -u 1000:1000 example/image', + 'docker run -u root:root example/image', + 'curl -u admin:password https://api.internal.corp/v1/x', + 'curl -u admin:${API_PASSWORD} https://api.internal.corp/v1/x', + ]; + + for (const line of CLEAN_ENV_LINES) { + it(`ignores: ${line}`, () => { + expect(scanText('app/.env.example', `${line}\n`)).toEqual([]); + }); + } + + it('a docker user and group pair is not an HTTP password, even when the next line mentions curl', () => { + const name = ['a', 'pp'].join(''); + expect(scanText('docs/run.md', `docker run -u ${name}:${name} img\ncurl https://api.internal.corp/x\n`)).toEqual([]); + }); + + it('ignores GitHub Actions secret references in YAML', () => { + const yaml = ['env:', ' NPM_TOKEN: ${{ secrets.NPM_TOKEN }}', ' API_KEY: "${{ secrets.API_KEY }}"', ''].join('\n'); + expect(scanText('.github/workflows/x.yml', yaml)).toEqual([]); + }); + + it('ignores label-like values and prose in JSON, YAML and Markdown', () => { + const json = ['{', ' "password": "Password",', ' "forgotPassword": "Forgot your password?",', ' "token": "Bearer"', '}', ''].join('\n'); + expect(scanText('src/i18n/en.json', json)).toEqual([]); + const md = ['**Password:** must be at least 8 characters', '- Token: required for every request', 'Set JWT_SECRET to a long random value', ''].join('\n'); + expect(scanText('docs/API.md', md)).toEqual([]); + }); + + it('ignores env references and empty fallbacks in source code', () => { + const code = [ + 'const a = process.env.JWT_SECRET;', + "const b = process.env.JWT_SECRET || '';", + 'const c = process.env.JWT_SECRET || (isDev ? crypto.randomBytes(32).toString("hex") : null);', + "const d = process.env.PORT || '3000';", + 'const e = process.env.API_TOKEN || `${other}`;', + 'const f = import.meta.env.VITE_FIREBASE_API_KEY;', + "const g = process.env['JWT_SECRET'];", + "const { JWT_SECRET, PORT = '3000' } = process.env;", + 'const h = jwt.sign(payload, secret, { expiresIn: "1h" });', + "const i = jwt.verify(token, process.env.JWT_SECRET, { algorithms: ['HS256'] });", + '', + ].join('\n'); + expect(scanText('server/server.js', code)).toEqual([]); + }); + + it('ignores ordinary identifiers and fixtures assigned to secret-like names in source code', () => { + const code = [ + "const TOKEN_KEY = 'auth_token';", + "const options = { credentials: 'same-origin', tokenUrl: 'https://auth.internal.test/token' };", + "const user = { password: 'Password123!', token: 'test-token-12345' };", + "const secretName = 'fish-db-credentials';", + "const label = { passwordPlaceholder: 'Enter your password' };", + "const PASSWORD_RULES = 'must contain a number and a symbol';", + "const authToken = 'ThisIsAVeryLongCamelCaseIdentifierNameHere';", + '', + ].join('\n'); + expect(scanText('src/lib/auth.js', code)).toEqual([]); + }); + + it('ignores placeholder tokens for the prefix-based rules', () => { + const text = [ + `aws AKIA${'IOSFODNN7'}${'EXAMPLE'}`, + `github ${['gh', 'p_'].join('')}${'x'.repeat(36)}`, + `stripe ${['sk_', 'live_'].join('')}${'X'.repeat(24)}`, + `google ${['AIza', 'x'.repeat(35)].join('')}`, + `neon ${['na', 'pi_'].join('')}${'x'.repeat(40)}`, + `neon ${['np', 'g_'].join('')}${'x'.repeat(12)}`, + `stack ${['ss', 'k_'].join('')}${'x'.repeat(32)}`, + `jwt ${['eyJ', 'hbGciOiJIUzI1NiJ9'].join('')}.${['eyJ', 'zdWIiOiJ1c2VyIn0'].join('')}.${'x'.repeat(30)}`, + '', + ].join('\n'); + expect(scanText('notes.md', text)).toEqual([]); + }); + + it('ignores a PEM header that is not followed by key material', () => { + const header = ['-----BEGIN ', 'PRIVATE KEY-----'].join(''); + expect(scanText('docs/x.md', `${header}\n...\n`)).toEqual([]); + expect(scanText('docs/x.md', `Keys start with ${header}.\n`)).toEqual([]); + }); + + it('ignores a JWT with a short signature and a truncated sample', () => { + const head = base64url({ alg: 'RS256', kid: 'abc' }); + const body = base64url({ email: 'a@b.test', sub: '1' }); + expect(scanText('docs/API.md', `Bearer ${[head, body, 'c2lnbg'].join('.')}\n`)).toEqual([]); + expect(scanText('docs/API.md', `{"token": "${[head, body].join('.').slice(0, 39)}..."}\n`)).toEqual([]); + }); + + it('ignores SQL with placeholder passwords', () => { + const sql = ["ALTER ROLE app WITH PASSWORD '';", "CREATE ROLE app LOGIN PASSWORD 'changeme';", "CREATE USER a IDENTIFIED BY 'your_password';", ''].join('\n'); + expect(scanText('db/rotate.sql', sql)).toEqual([]); + }); + + it('a bare `password' + " '...'` is only SQL in a .sql file", () => { + const value = randomString(20, 43); + expect(scanText('docs/x.md', `the password '${value}' was shown\n`)).toEqual([]); + expect(scanText('db/x.sql', `SET password '${value}';\n`).map((f) => f.rule)).toEqual(['sql-password-literal']); + }); +}); + +describe('placeholder markers are anchored, not substring tests', () => { + const realish = randomString(14, 60); + const tail = randomString(12, 61); + const name = ['DB_', 'PASSWORD'].join(''); + const dbScheme = ['postgres', 'ql'].join(''); + const flagged = (value) => scanText('app/.env.production', `${name}=${value}\n`).map((f) => f.rule); + + it('a `$` followed by letters is a reference only when the whole value is an upper-case variable name', () => { + expect(isPlaceholder('$DB_PASSWORD')).toBe(true); + expect(isPlaceholder('$db_password')).toBe(true); + expect(isPlaceholder(`$${randomString(15, 62)}`)).toBe(false); + expect(flagged(`$${randomString(15, 62)}`)).toEqual(['secret-assignment']); + const url = `DATABASE_URL=${dbScheme}://admin:$${randomString(15, 63)}@db.internal.corp/db\n`; + expect(scanText('.env', url).map((f) => f.rule)).toEqual(['url-password']); + }); + + it('a marker inside a long random value does not make it a placeholder', () => { + for (const marker of ['...', '***', '___', '', 'xxx']) { + expect(isPlaceholder(`${realish}${marker}${tail}`), marker).toBe(false); + expect(flagged(`${realish}${marker}${tail}`), marker).toEqual(['secret-assignment']); + } + }); + + it('a marker that stands for most of the value still is a placeholder', () => { + for (const value of ['AIza...', 'xxxxxxxxxxxxxxxx', 'sk_live_xxxxxxxxxxxx', '', '[YOUR-KEY]', '********', 'abcd...wxyz', 'eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...']) { + expect(isPlaceholder(value), value).toBe(true); + } + }); + + it('human-style passwords that contain a word like change/sample/enter/todo/foo are NOT placeholders', () => { + const values = ['Change2024!', 'Sample123', 'Winter2024here', 'Fake1234', 'Spring_2024_todo', 'Enter2024', 'Mock1234!', 'Banana_bar_9', 'Admin_foo_1']; + for (const value of values) { + expect(isPlaceholder(value), value).toBe(false); + expect(flagged(value), value).toEqual(['secret-assignment']); + const url = `DATABASE_URL=${dbScheme}://admin:${value}@db.internal.corp/db\n`; + expect(scanText('.env', url).map((f) => f.rule), value).toEqual(['url-password']); + } + }); + + it('real placeholders built from those words still pass', () => { + for (const value of ['your_password', 'change-me', 'change_me_please', 'replace-with-your-key', 'enter_your_password', 'paste-token-here', 'fake_token_for_local_dev_1', 'foo_bar', 'new_password', 'todo']) { + expect(isPlaceholder(value), value).toBe(true); + } + }); +}); + +describe('sample values are scoped to the file that shows them', () => { + const sample = ['secure', 'Password', '123'].join(''); + const dbScheme = ['postgres', 'ql'].join(''); + + it('is ignored in docs/API.md (exact match only)', () => { + expect(scanText('docs/API.md', `{"password": "${sample}"}\n`)).toEqual([]); + expect(scanText('docs/API.md', `{"password": "${sample}4"}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + }); + + it('is NOT ignored anywhere else', () => { + const name = ['DB_', 'PASSWORD'].join(''); + expect(scanText('app/.env.production', `${name}=${sample}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + expect(scanText('.env', `${['JWT_', 'SECRET'].join('')}=${sample}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + expect(scanText('.env', `DATABASE_URL=${dbScheme}://owner:${sample}@db.internal:5432/app\n`).map((f) => f.rule)).toEqual(['url-password']); + const mongo = `const uri = "${['mongodb'].join('')}://app:${sample}@cluster0.internal/app";\n`; + expect(scanText('app.js', mongo).map((f) => f.rule)).toEqual(['url-password']); + }); +}); + +describe('quoted values with whitespace', () => { + const name = ['JWT_', 'SECRET'].join(''); + const value = randomString(16, 71); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + + it('flags a quoted passphrase that contains spaces (double, single and backtick quotes)', () => { + for (const q of ['"', "'", '`']) { + expect(rules('app/.env.x', `${name}=${q}random long password ${value}!${q}\n`), q).toEqual(['secret-assignment']); + } + expect(rules('deploy.yaml', `${name}: "random long password ${value}!"\n`)).toEqual(['secret-assignment']); + }); + + it('trims whitespace inside the quotes before judging the value', () => { + expect(rules('app/.env.x', `${name}=" ${value} "\n`)).toEqual(['secret-assignment']); + expect(rules('app/.env.x', `${name}=" "\n`)).toEqual([]); + }); + + it('still flags a quoted value without spaces', () => { + expect(rules('app/.env.x', `${name}="${value}"\n`)).toEqual(['secret-assignment']); + }); + + it('an UNQUOTED value with spaces runs to the end of the line in dotenv, YAML and ini, but not in shell scripts', () => { + expect(rules('app/.env.x', `${name}=random long password ${value}\n`)).toEqual(['secret-assignment']); + expect(rules('c.yml', `${name.toLowerCase()}: random long password ${value}\n`)).toEqual(['secret-assignment']); + expect(rules('c.ini', `${name.toLowerCase()} = random long password ${value}\n`)).toEqual(['secret-assignment']); + // In a shell script the words after the first are a command, not part of the value. + expect(rules('run.sh', `${name}=random long password ${value}\n`)).toEqual([]); + }); + + it('quoted placeholders with spaces still pass', () => { + expect(rules('app/.env.x', `${name}="your secret goes here"\n`)).toEqual([]); + expect(rules('app/.env.x', `${name}="change me to something long"\n`)).toEqual([]); + }); +}); + +describe('shell parameter expansions in config files', () => { + const name = ['JWT_', 'SECRET'].join(''); + const weak = ['APP_', 'TOKEN'].join(''); + const value = randomString(16, 72); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + const ops = [':-', '-', ':=', '=', ':+', '+']; + + it('flags a literal default or alternate operand for every operator', () => { + for (const op of ops) { + expect(rules('docker.env', `${name}=\${${name}${op}${value}}\n`), op).toEqual(['secret-assignment']); + expect(rules('docker-compose.yml', ` - ${name}=\${${name}${op}${value}}\n`), `compose ${op}`).toEqual(['secret-assignment']); + } + expect(rules('docker.env', `${name}="\${${name}:-${value}}"\n`)).toEqual(['secret-assignment']); + expect(rules('docker.env', `${weak}=\${${weak}:-${value}}\n`)).toEqual(['secret-assignment']); + }); + + it('flags a literal hidden inside a nested expansion', () => { + expect(rules('docker.env', `${name}=\${OUTER:-\${${name}:-${value}}}\n`)).toEqual(['secret-assignment']); + expect(rules('docker.env', `${name}=\${OUTER:-\${INNER}}\n`)).toEqual([]); + }); + + it('does not flag pure substitutions or placeholder/variable operands', () => { + for (const text of [ + `\${${name}}`, + `$${name}`, + `\${${name}:?set ${name} in the environment}`, + `\${${name}:-}`, + `\${${name}:-changeme}`, + `\${${name}:-your_secret_here}`, + `\${${name}:-\${OTHER_SECRET}}`, + `\${${name}:-$OTHER_SECRET}`, + `\${${name}:-\${A:-\${B}}}`, + ]) { + expect(rules('docker.env', `${name}=${text}\n`), text).toEqual([]); + } + }); + + it('checks the operand of an unbalanced expansion and stays fast on hostile nesting', SLOW, () => { + expect(rules('docker.env', `${name}=\${${name}:-${value}\n`)).toEqual(['secret-assignment']); + expect(rules('docker.env', `${name}=\${${name}:-\${OTHER}\n`)).toEqual([]); + const hostile = `${name}=${'${A:-'.repeat(20000)}\n`; + const started = Date.now(); + scanText('docker.env', hostile); + expect(Date.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); + + it('is limited to config files: a JS template literal is still not a secret', () => { + expect(rules('app.js', `const ${name} = \`\${${name}:-${value}}\`;\n`)).toEqual([]); + }); +}); + +describe('review round 3: expansions, phrases, continuations and escapes', () => { + const name = ['JWT_', 'SECRET'].join(''); + const lower = name.toLowerCase(); + const token = randomString(16, 91); + const phrase = ['correct', 'horse', 'battery', 'staple'].join(' '); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + const HIT = ['secret-assignment']; + + it('a passphrase with spaces behind a :- default is a finding, quoted or bare', () => { + expect(rules('a.sh', `${name}="\${A:-${phrase}}"\n`)).toEqual(HIT); + expect(rules('a.env', `${name}=\${A:-${phrase}}\n`)).toEqual(HIT); + expect(rules('a.env', `${name}=\${A:-"${phrase}"}\n`)).toEqual(HIT); + expect(rules('a.env', `${name}="${phrase}"\n`)).toEqual(HIT); + }); + + it('a literal glued to an expansion is a finding, a file path built from one is not', () => { + expect(rules('a.env', `${name}=\${A}${token}\n`)).toEqual(HIT); + expect(rules('a.env', `${name}=${token}\${SUFFIX}\n`)).toEqual(HIT); + expect(rules('a.env', `${name}=\${A}${token}\${B}\n`)).toEqual(HIT); + expect(rules('a.env', `${name}=/run/secrets/\${NAME}\n`)).toEqual([]); + expect(rules('a.env', `${name}=\${A} # note about it\n`)).toEqual([]); + }); + + it('fails closed past the nesting cap and for an unterminated expansion', () => { + for (const depth of [9, 10, 20]) { + const nested = Array.from({ length: depth }, (_, i) => `\${V${i}:-`).join('') + token + '}'.repeat(depth); + expect(rules('a.env', `${name}=${nested}\n`), `depth ${depth}`).toEqual(HIT); + } + expect(rules('a.env', `${name}=\${A:-${token}\n`)).toEqual(HIT); + }); + + it('message-catalog sentences under secret-like keys are not findings', () => { + for (const [file, text] of [ + ['en.json', '{"token": "Invalid or expired token"}'], + ['en.json', '{"apiKey": "API key is missing"}'], + ['en.json', '{"password": "Please choose a password"}'], + ['en.json', '{"secret": "Secret is invalid"}'], + ['en.json', '{"password": "Password must be at least 8 characters"}'], + ['en.yml', 'password: "Passwords do not match"'], + ['en.yml', 'password: Password is required'], + ['messages.yml', 'password: "Password is required"'], + ]) { + expect(rules(file, `${text}\n`), text).toEqual([]); + } + // A random-looking word inside a phrase still counts. + expect(rules('en.json', `{"password": "the code is ${token}"}\n`)).toEqual(HIT); + }); + + it('unquoted passphrases in YAML and ini, and YAML block scalars, are findings', () => { + expect(rules('c.yml', `${lower}: ${phrase}\n`)).toEqual(HIT); + expect(rules('c.ini', `${lower} = ${phrase}\n`)).toEqual(HIT); + expect(rules('c.yml', `${lower}: ${phrase} # not a comment part\n`)).toEqual(HIT); + for (const indicator of ['|', '>', '|-', '>-']) { + expect(rules('c.yml', `${lower}: ${indicator}\n ${token}${token}\nother: 1\n`), indicator).toEqual(HIT); + } + expect(rules('messages.yml', `${lower}: |\n Passwords do not match\nother: 1\n`)).toEqual([]); + expect(rules('c.yml', `${lower}: |\nother: ${token}${token}\n`)).toEqual([]); + expect(rules('c.yml', `${lower}: !vault |\n`)).toEqual([]); + }); + + it('an escaped quote near the start of a quoted value does not hide the rest', () => { + for (const text of [`{"${lower}": "a\\"${token}${token}"}`, `${lower} = 'a\\'${token}${token}'`]) { + expect(rules('c.json', `${text}\n`), text).toEqual(HIT); + } + }); + + it('Makefile ?= and +=, and Dockerfile ENV/ARG with a space, are recognised', () => { + expect(rules('Makefile', `${name} ?= ${token}\n`)).toEqual(HIT); + expect(rules('Makefile', `${name} += ${token}\n`)).toEqual(HIT); + expect(rules('Dockerfile', `ENV ${name} ${token}\n`)).toEqual(HIT); + expect(rules('Dockerfile', `ARG ${name} ${token}\n`)).toEqual(HIT); + expect(rules('Dockerfile', `ENV ${name} \${X:-${token}}\n`)).toEqual(HIT); + expect(rules('Dockerfile', `ENV ${name} changeme\n`)).toEqual([]); + expect(rules('Dockerfile', `ENV ${name} \${${name}}\n`)).toEqual([]); + expect(rules('app.js', `ENV ${name} ${token}\n`)).toEqual([]); + }); +}); + +describe('documentation sample is compared as the whole value', () => { + const sample = ['secure', 'Password', '123'].join(''); + const real = randomString(16, 73); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + + it('the exact sample passes in docs/API.md in every quoting style', () => { + expect(rules('docs/API.md', ` "password": "${sample}"\n`)).toEqual([]); + expect(rules('docs/API.md', `-d '{"username":"fisherman_joe","password":"${sample}"}'\n`)).toEqual([]); + expect(rules('docs/API.md', `PASSWORD=${sample}\n`)).toEqual([]); + expect(rules('docs/API.md', `password: '${sample}'\n`)).toEqual([]); + }); + + it('the sample glued to other text is still a finding', () => { + for (const text of [ + `PASSWORD=${sample}!${real}`, + `PASSWORD=prefix-${sample}-suffix`, + `PASSWORD=${real}=${sample}`, + `PASSWORD=${real}:${sample}`, + `"password": "prefix ${sample}"`, + `"password": "${sample} ${real}"`, + `PASSWORD=${sample}${real}`, + ]) { + expect(rules('docs/API.md', `${text}\n`), text).toEqual(['secret-assignment']); + } + }); + + it('the exact sample is not exempt in any other file', () => { + expect(rules('docs/OTHER.md', `PASSWORD=${sample}\n`)).toEqual(['secret-assignment']); + expect(rules('app/docs/API.md', `PASSWORD=${sample}\n`)).toEqual(['secret-assignment']); + }); +}); + +describe('review round 5: URL password expansions and quoted property names', () => { + const value = randomString(20, 501); + const name = secretName('JWT_', 'SECRET'); + const scheme = ['post', 'gresql'].join(''); + const urlLine = (password, file = '.env.prod') => scanText(file, `DATABASE_URL=${scheme}://user:${password}@prod.internal/db\n`).map((f) => f.rule); + const ref = (n) => `$` + `{${n}}`; + const withDefault = (n, literal) => `$` + `{${n}:-${literal}}`; + + it('flags a literal default inside an expansion in a URL password', () => { + expect(urlLine(withDefault('DB_PASSWORD', value))).toEqual(['url-password']); + expect(urlLine(`$` + `{DB_PASSWORD-${value}}`)).toEqual(['url-password']); + expect(urlLine(`$` + `{DB_PASSWORD:=${value}}`)).toEqual(['url-password']); + expect(urlLine(`$` + `{DB_PASSWORD:+${value}}`)).toEqual(['url-password']); + }); + + it('flags a literal nested in an expansion, and literal text next to a reference', () => { + expect(urlLine(`$` + `{OUTER:-` + withDefault('INNER', value) + `}`)).toEqual(['url-password']); + expect(urlLine(`${ref('DB_PASSWORD')}${value}`)).toEqual(['url-password']); + expect(urlLine(`${value}${ref('DB_PASSWORD')}`)).toEqual(['url-password']); + }); + + it('flags a literal default in a curl -u password too', () => { + const line = `curl -u admin:${withDefault('API_PW', value)} https://api.internal/x\n`; + expect(scanText('deploy.sh', line).map((f) => f.rule)).toEqual(['url-password']); + expect(scanText('deploy.sh', `curl -u admin:${ref('API_PW')} https://api.internal/x\n`)).toEqual([]); + }); + + it('negative controls: a pure reference, or a placeholder default, still passes', () => { + expect(urlLine(ref('DB_PASSWORD'))).toEqual([]); + expect(urlLine('$DB_PASSWORD')).toEqual([]); + expect(urlLine(`$` + `{DB_PASSWORD:?set it}`)).toEqual([]); + expect(urlLine(withDefault('DB_PASSWORD', 'changeme'))).toEqual([]); + expect(urlLine(withDefault('DB_PASSWORD', ref('OTHER')))).toEqual([]); + expect(urlLine(`${ref('A')}:${ref('B')}`.replace(':', ''))).toEqual([]); + }); + + it('does not let a long user name or password stop the URL match', () => { + const longUser = randomString(200, 502, LOWER); + const longPassword = randomString(900, 503); + expect(scanText('.env.prod', `U=${scheme}://${longUser}:${value}@prod.internal/db\n`).map((f) => f.rule)).toEqual(['url-password']); + expect(scanText('.env.prod', `U=${scheme}://user:${longPassword}@prod.internal/db\n`).map((f) => f.rule)).toEqual(['url-password']); + expect(scanText('.env.prod', `U=${scheme}://user:${'x'.repeat(900)}@prod.internal/db\n`)).toEqual([]); + }); + + it('flags a secret-like name assigned through a quoted property', () => { + const q = { single: "'", double: '"', backtick: '`' }; + for (const [label, quote] of Object.entries(q)) { + const bracket = `config[${quote}${name}${quote}] = ${quote}${value}${quote};\n`; + expect(scanText('config.js', bracket).map((f) => f.rule), label).toEqual(['secret-assignment']); + } + for (const text of [ + `obj?.['${name}'] = '${value}';\n`, + `a.b['c']['${name}'] = '${value}';\n`, + `a['b'].c["${name}"] = "${value}";\n`, + `config[ '${name}' ] = '${value}';\n`, + `module.exports['${name}'] = '${value}'\n`, + `const o = { ['${name}']: '${value}' };\n`, + `settings["${name}"] := "${value}"\n`, + ]) { + expect(scanText('config.ts', text).map((f) => f.rule), text.replace(value, 'V')).toEqual(['secret-assignment']); + } + }); + + it('applies the same value rules to bracket notation', () => { + expect(scanText('config.js', `config['${name}'] = 'your_secret_here';\n`)).toEqual([]); + expect(scanText('config.js', `config['${name}'] = process.env.${name};\n`)).toEqual([]); + expect(scanText('config.js', `config['${name}'] = getSecret();\n`)).toEqual([]); + expect(scanText('config.js', `config['title'] = '${value}';\n`)).toEqual([]); + expect(scanText('config.js', `x = ['${name}', '${value}'];\n`)).toEqual([]); + }); + + it('still sees a secret name and a fallback literal longer than the old length caps', () => { + const longName = `${'A_'.repeat(60)}${name}`; + expect(scanText('.env', `${longName}=${value}\n`).map((f) => f.rule)).toEqual(['secret-assignment']); + const longFallback = randomString(700, 504); + expect(scanText('app.js', `const s = process.env.${name} || '${longFallback}';\n`).map((f) => f.rule)).toEqual(['hardcoded-secret-fallback']); + }); +}); + +describe('review round 6: a rejected assignment must not hide the one after it', () => { + const value = randomString(22, 601); + const name = secretName('JWT_', 'SECRET'); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + const shapes = { + 'py dict after a non-secret bracket assignment': ['s.py', `settings["auth"] = {"${name}": "${value}"}\n`], + 'py dict() after a bracket assignment': ['s.py', `CONFIG["auth"] = dict(${name}="${value}")\n`], + 'py single quotes, no spaces': ['s.py', `cfg['auth']={'${name}':'${value}'}\n`], + 'js object with an unquoted key': ['s.js', `cfg['auth'] = {${name}: '${value}'};\n`], + 'minified JSON': ['c.json', `{"port":3000,"${name}":"${value}"}\n`], + 'nested minified JSON': ['c.json', `{"a":{"${name}":"${value}"}}\n`], + 'one-line JSON env block': ['c.json', `{"env":{"${name}":"${value}"}}\n`], + 'statement after a statement': ['a.js', `x=1;${name}='${value}';\n`], + 'object literal without spaces': ['a.js', `c = {'${name}': '${value}'};\n`], + 'module.exports object': ['a.js', `module.exports={${name}:'${value}'};\n`], + 'call arguments': ['a.js', `f(x=1,${name}='${value}');\n`], + 'bracket then bracket': ['a.js', `env['PORT']=3;env['${name}']='${value}';\n`], + 'chained bracket assignment': ['a.js', `cfg['x']=cfg['${name}']='${value}';\n`], + 'YAML flow mapping': ['a.yml', `env: {PORT: 3, ${name}: ${value}}\n`], + 'inside a quoted value of a non-secret key': ['a.yml', `run: "${name}=${value} node app.js"\n`], + }; + + it.each(Object.entries(shapes))('flags a secret in: %s', (_label, [file, text]) => { + expect(rules(file, text)).toContain('secret-assignment'); + }); + + it.each(Object.entries(shapes))('a placeholder value passes in: %s', (_label, [file, text]) => { + expect(rules(file, text.replace(value, 'your_secret_here'))).toEqual([]); + }); + + it('does not treat an expansion default or a URL user as a second assignment', () => { + const ref = `$` + `{OTHER_SECRET}`; + expect(rules('docker.env', `${name}=$` + `{${name}:-${ref}}\n`)).toEqual([]); + expect(rules('a.env', `DATABASE_URL=${['post', 'gresql'].join('')}://password:${value}@prod.internal/db\n`)).toEqual(['url-password']); + }); +}); + +describe('isPlaceholder', () => { + it.each([ + '', + ' ', + 'password', + 'PASSWORD', + 'secret_key', + 'your_api_key', + 'change-me', + 'changeme', + 'example', + 'xxxx', + 'REDACTED', + '', + '[YOUR-PASSWORD]', + '${TOKEN}', + '$TOKEN', + '%TOKEN%', + '{{ token }}', + '...', + 'process.env.TOKEN', + 'import.meta.env.VITE_TOKEN', + '--', + 'AKIAIOSFODNN7EXAMPLE', + ])('treats %j as a placeholder', (value) => { + expect(isPlaceholder(value)).toBe(true); + }); + + it('does not treat random-looking values as placeholders', () => { + expect(isPlaceholder(randomString(32, 51))).toBe(false); + expect(isPlaceholder(randomString(20, 52, UPPER_ALNUM))).toBe(false); + }); + + it('never swallows a random value, whatever it contains (4000 samples per alphabet)', SLOW, () => { + for (const alphabet of [ALNUM, PASSWORD_MANAGER, BASE64, HEX, LOWER_ALNUM]) { + let swallowed = 0; + for (let i = 0; i < 4000; i += 1) { + const value = randomString(16 + (i % 17), 5000 + i * 13, alphabet); + // The little generator sometimes degenerates into runs of one character; that is not a random secret. + if (new Set(value).size < 8) continue; + if (isPlaceholder(value)) swallowed += 1; + } + expect(swallowed).toBe(0); + } + }); +}); + +// --------------------------------------------------------------------------- +// Hosts +// --------------------------------------------------------------------------- + +describe('url-password hosts', () => { + const pw = randomString(20, 70); + const dbScheme = ['postgres', 'ql'].join(''); + const rules = (host) => scanText('.env', `DATABASE_URL=${dbScheme}://admin:${pw}@${host}/db\n`).map((f) => f.rule); + + it.each(['db.example.com', 'example.org', 'a.b.example.net', 'svc.example', 'x.test', 'y.invalid', 'localhost:5432', '127.0.0.1'])( + 'lets reserved documentation and loopback host %s pass', + (host) => { + expect(rules(host)).toEqual([]); + }, + ); + + it.each([ + 'example-prod-db.c1abc.us-east-1.rds.amazonaws.com', + 'ep-example-app-123456.us-east-2.aws.neon.tech', + 'prod.example-corp.io', + 'example.internal', + 'my.example.co', + 'db.internal.corp', + ])('does NOT let host %s hide a password', (host) => { + expect(rules(host)).toEqual(['url-password']); + }); + + it('flags a real-looking password for every user name (the user side is not a placeholder test)', () => { + for (const user of ['user', 'username', 'example', 'password', 'root', 'postgres', 'admin', 'your_user', '']) { + const text = `DATABASE_URL=${dbScheme}://${user}:${pw}@db.internal.corp/db\n`; + expect(scanText('.env', text).map((f) => f.rule), user).toEqual(['url-password']); + } + }); +}); + +// --------------------------------------------------------------------------- +// Redaction +// --------------------------------------------------------------------------- + +describe('redaction', () => { + it('findings carry only path, line and rule', () => { + for (const c of SECRET_CASES) { + for (const finding of scanText(c.path, c.text)) { + expect(Object.keys(finding).sort(), c.name).toEqual(['line', 'path', 'rule']); + } + } + }); + + it('the report never contains the matched text, whole or in part', () => { + for (const c of SECRET_CASES) { + const report = formatReport(scanText(c.path, c.text)); + expect(report, c.name).toContain(`${c.path}:`); + expect(report, c.name).toContain(c.rule); + expect(report, c.name).not.toContain(c.secret); + for (const piece of windows(c.secret)) expect(report, c.name).not.toContain(piece); + } + }); + + it('a report over all cases together leaks nothing', () => { + const all = SECRET_CASES.flatMap((c) => scanText(c.path, c.text)); + const report = formatReport(all); + for (const c of SECRET_CASES) { + for (const piece of windows(c.secret)) expect(report).not.toContain(piece); + } + }); + + it('strips control characters from printed paths', () => { + const report = formatReport([{ path: 'a\nb\u001b[31m', line: 1, rule: 'jwt-token' }]); + expect(report.includes(String.fromCharCode(27))).toBe(false); + expect(report.split('\n').some((l) => l.startsWith('b'))).toBe(false); + }); + + it('the history report holds only commit, path, rule and counts', () => { + const report = formatHistoryReport([{ commit: 'a'.repeat(40), path: '.env', rule: 'jwt-token', count: 2 }], { + commits: 5, + shallow: true, + oversize: 1, + }); + expect(report).toContain('aaaaaaa .env jwt-token x2'); + expect(report).toContain('shallow clone'); + expect(report).toContain('1 file version NOT scanned'); + }); + + it('the oversize report names paths only', () => { + expect(formatOversizeReport(['data/big.json'])).toContain('data/big.json'); + }); +}); + +// --------------------------------------------------------------------------- +// Scan time stays linear on hostile input +// --------------------------------------------------------------------------- + +// --------------------------------------------------------------------------- +// Review round 7 (finding 1): a sentence is not the same as "contains one common word" +// +// Class: quoted (or rest-of-line) values with spaces under a strong secret name. Sentence-likeness is judged on the +// whole text: plain words only, mostly sentence vocabulary, written like a sentence. Both sides are listed on purpose. +// --------------------------------------------------------------------------- + +describe('review round 7 (1): passphrase versus UI sentence', () => { + const name = ['JWT_', 'SECRET'].join(''); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + const digits = randomString(3, 201, DIGITS); + + // Passphrases that contain ordinary words (several of them prose words) and must be reported. + const PASSPHRASES = [ + 'correct horse battery and staple', // one function word ("and") + `random long password ${digits}!`, // "long" is a prose word; the digit-symbol piece is credential-shaped + 'this is my super secret passphrase', // three prose words of six, written in lower case + 'Correct Horse Battery Staple', // Title Case diceware + 'correct horse battery staple.', // closing punctuation does not make it a sentence + 'the quick brown fox jumps over the lazy dog', // 3 of 9 words, lower case + `Tr0ub4dor and Horse Battery ${digits}`, // letters mixed with digits + `hunter${digits} is the password`, // "hunter123": mixed piece + `my dog is named Rover ${digits}`, // two prose words of five + 'purple monkey dishwasher and the tortoise', // 2 of 5 + 'Open sesame, said the ancient door', + `winter-${digits} is coming to the north`, // "winter-123" is not a word-number compound + 'Password1! and more', // symbol-mixed piece + 'blue::green::red the sky', // symbol runs between words + ]; + + it.each(PASSPHRASES)('reports the quoted passphrase %j in env, YAML, JSON and ini files', (phrase) => { + expect(rules('app/.env.x', `${name}="${phrase}"\n`)).toEqual(['secret-assignment']); + expect(rules('app/.env.x', `${name}='${phrase}'\n`)).toEqual(['secret-assignment']); + expect(rules('c.yaml', `${name.toLowerCase()}: "${phrase}"\n`)).toEqual(['secret-assignment']); + expect(rules('c.json', `{"${name}": "${phrase}"}\n`)).toEqual(['secret-assignment']); + expect(rules('c.ini', `${name.toLowerCase()} = "${phrase}"\n`)).toEqual(['secret-assignment']); + }); + + it('reports an unquoted passphrase (the rest of a YAML/ini/dotenv line) the same way', () => { + expect(rules('c.yaml', `${name.toLowerCase()}: correct horse battery and staple\n`)).toEqual(['secret-assignment']); + expect(rules('app/.env.x', `${name}=correct horse battery and staple\n`)).toEqual(['secret-assignment']); + }); + + // Real UI, validation and error text that message catalogs keep under secret-looking keys. + const SENTENCES = [ + 'Your session token has expired, please sign in again.', + 'Invalid or expired token', + 'Passwords do not match', + 'Password must be at least 8 characters', + 'Forgot password?', + 'Enter your password', + 'Please enter a valid token', + 'Reset your password', + 'Unable to reach the server, try again later', + 'The token %s has expired', + 'Token {name} is invalid', + 'Enter the 6-digit code we sent to your email', + 'The :attribute must be at least 8 characters.', + 'Your password has been changed successfully.', + 'This link is no longer valid. Request a new one.', + 'password too short', + 'passwords do not match', + 'La sesión ha caducado, inicia sesión de nuevo', // another language, with non-ASCII letters + ]; + + it.each(SENTENCES)('does not report the UI sentence %j', (sentence) => { + for (const key of ['resetToken', 'password', 'invalid_token']) { + expect(rules('locales/en.json', `{"${key}": "${sentence}"}\n`), key).toEqual([]); + expect(rules('messages.yml', `${key}: "${sentence}"\n`), key).toEqual([]); + expect(rules('messages.properties', `${key}=${sentence}\n`), key).toEqual([]); + } + }); + + it('a random-looking word still makes a sentence-shaped value a finding', () => { + expect(rules('app/.env.x', `${name}="The token is ${randomString(20, 202)} today."\n`)).toEqual(['secret-assignment']); + }); + + it('a passphrase made only of function words is a finding outside a message catalog, whatever it looks like', () => { + for (const phrase of ['to be or not to be', 'It is what it is.', 'This is the way.', 'My voice is my password.', 'May the force be with you.']) { + expect(rules('app/.env.x', `${name}="${phrase}"\n`), phrase).toEqual(['secret-assignment']); + } + }); + + it('the same sentences are prose inside a message catalog (locales, i18n, messages, en.json)', () => { + for (const file of ['locales/en.json', 'src/i18n/app.json', 'messages.properties', 'en.yml']) { + expect(rules(file, file.endsWith('.json') ? `{"${name}": "It is what it is."}\n` : `${name.toLowerCase()}: "It is what it is."\n`), file).toEqual([]); + } + }); + + it('documents the tradeoff: an ASCII sentence in another language is reported (use a placeholder or the allow marker)', () => { + expect(rules('locales/es.json', `{"resetToken": "El token ha caducado"}\n`)).toEqual(['secret-assignment']); + expect(rules('locales/es.json', `{"resetToken": "El token ha caducado" } // ${ALLOW_MARKER}\n`)).toEqual([]); + }); + + it('weak names are unaffected: a spaced value there needs a random-looking word', () => { + expect(rules('app/.env.x', `TOKEN_HINT="correct horse battery and staple"\n`)).toEqual([]); + expect(rules('app/.env.x', `TOKEN_HINT="hint ${randomString(20, 203)}"\n`)).toEqual(['secret-assignment']); + }); +}); + +// --------------------------------------------------------------------------- +// Review round 7 (finding 2): native credential-file formats +// +// Class: files whose syntax is not NAME=value. Siblings enumerated: .netrc/_netrc (one line, multi-line, default, +// account, quoted, CRLF), .pgpass (wildcards, escaped colons), .git-credentials (password, token as user, doc host), +// .npmrc/.yarnrc (=, scoped //registry/:key, _auth, _password, yarn "key" "value", yarnrc.yml npmAuth*), +// .pypirc, .aws/credentials, docker config.json/.dockercfg/.dockerconfigjson, kubeconfig, .htpasswd/.htdigest, +// .my.cnf, .s3cfg/.boto, Terraform (.terraformrc, tfrc.json, tfvars, tfstate), .curlrc, .wgetrc, .vault-token. +// --------------------------------------------------------------------------- + +describe('review round 7 (2): credential-file formats', () => { + const v = randomString(24, 211); + const tiny = ['ab', 'c'].join(''); // too short for a credential file value + const hex = randomString(40, 212, HEX); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + + // [path, content, rule that must fire] + const FINDINGS = [ + ['.netrc', `machine api.internal login app password ${v}\n`, 'credential-file'], + ['home/_netrc', `machine api.internal\n login app\n password ${v}\n`, 'credential-file'], + ['.netrc', `default login app password ${v}\r\n`, 'credential-file'], + ['.netrc', `machine ftp.internal login app account ${v}\n`, 'credential-file'], + ['.netrc', `machine api.internal login app password "${v} tail"\n`, 'credential-file'], + ['.netrc', `machine a login u password ${tiny}\nmachine b login u password ${v}\n`, 'credential-file'], + ['.pgpass', `db.internal:5432:prod:app:${v}\n`, 'credential-file'], + ['.pgpass', `*:*:*:app:${v}\n`, 'credential-file'], + ['.pgpass', `db.internal:5432:prod:app:a\\:b${v}\r\n`, 'credential-file'], + ['pgpass.conf', `localhost:5432:*:postgres:${v}\n`, 'credential-file'], + ['.git-credentials', `https://user:${v}@example.com\n`, 'credential-file'], + ['.git-credentials', `https://${v}@github.com\n`, 'credential-file'], + ['.git-credentials', `https://oauth2:${v}@gitlab.internal.org\n`, 'url-password'], + ['.npmrc', `//registry.npmjs.org/:_authToken=${v}\n`, 'secret-assignment'], + ['.npmrc', `_authToken=${v}\n`, 'secret-assignment'], + ['.npmrc', `_auth=${v}\n`, 'secret-assignment'], + ['.npmrc', `//npm.internal.org/:_password=${v}\n`, 'secret-assignment'], + ['.npmrc', `//npm.internal.org/:_authToken=abcd12\n`, 'secret-assignment'], // short values count in credential files + ['.yarnrc', `"//registry.npmjs.org/:_authToken" "${v}"\n`, 'credential-file'], + ['.yarnrc.yml', `npmAuthToken: ${v}\n`, 'secret-assignment'], + ['.yarnrc.yml', `npmAuthIdent: user:${v}\n`, 'secret-assignment'], + ['.pypirc', `[pypi]\nusername = __token__\npassword = pypi-${v}\n`, 'secret-assignment'], + ['.aws/credentials', `[default]\naws_secret_access_key = ${v}\n`, 'secret-assignment'], + ['credentials', `[default]\naws_session_token = ${v}${v}\n`, 'secret-assignment'], + ['.aws/config', `[profile x]\naws_secret_access_key = ${v}\n`, 'secret-assignment'], + ['.docker/config.json', `{"auths":{"ghcr.io":{"auth":"${v}=="}}}\n`, 'secret-assignment'], + ['.docker/config.json', `{"identitytoken":"${v}"}\n`, 'secret-assignment'], + ['.docker/config.json', `{"auths":{"h":{"registrytoken":"${v}"}}}\n`, 'secret-assignment'], + ['.dockercfg', `{"https://index.docker.io/v1/":{"auth":"${v}"}}\n`, 'secret-assignment'], + ['manifests/pull-secret.yaml', `data:\n .dockerconfigjson: ${v}${v}\n`, 'credential-file'], + ['.kube/config', `users:\n- name: a\n user:\n token: ${v}\n`, 'secret-assignment'], + ['kubeconfig', `users:\n- user:\n password: ${v}\n`, 'secret-assignment'], + ['ci/dev.kubeconfig', `users:\n- user:\n client-key-data: ${v}${v}\n`, 'credential-file'], + ['deploy/cluster.yaml', `users:\n- user:\n client-key-data: ${v}${v}\n`, 'credential-file'], + ['.htpasswd', `admin:$apr1$${randomString(8, 213)}$${randomString(22, 214)}\n`, 'credential-file'], + ['.htpasswd', `admin:$2y$05$${randomString(53, 215)}\n`, 'credential-file'], + ['.htpasswd', `admin:{SHA}${randomString(28, 216)}\n`, 'credential-file'], + ['.htpasswd', `admin:${randomString(13, 217)}\r\n`, 'credential-file'], + ['.htdigest', `admin:private area:${hex.slice(0, 32)}\n`, 'credential-file'], + ['.my.cnf', `[client]\nuser=root\npassword=${v}\n`, 'secret-assignment'], + ['.my.cnf', `[client]\npassword = "${v}"\n`, 'secret-assignment'], + ['.my.cnf', `[client]\npassword=hunter22\n`, 'secret-assignment'], + ['.mylogin.cnf', `[client]\npassword=${v}\n`, 'secret-assignment'], + ['.s3cfg', `[default]\nsecret_key = ${v}\n`, 'secret-assignment'], + ['.s3cfg', `[default]\naccess_token = ${v}\n`, 'secret-assignment'], + ['.boto', `[Credentials]\ngs_secret_access_key = ${v}\n`, 'secret-assignment'], + ['.terraformrc', `credentials "app.terraform.io" {\n token = "${v}"\n}\n`, 'secret-assignment'], + ['credentials.tfrc.json', `{"credentials":{"app.terraform.io":{"token":"${v}"}}}\n`, 'secret-assignment'], + ['prod.tfvars', `db_password = "${v}"\n`, 'secret-assignment'], + ['terraform.tfstate', `{"attributes":{"password":"${v}"}}\n`, 'secret-assignment'], + ['.curlrc', `user = "name:${v}"\n`, 'credential-file'], + ['.curlrc', ['--user', ' ', 'name:', v, '\n'].join(''), 'credential-file'], + ['.wgetrc', `password = ${v}\n`, 'secret-assignment'], + ['.vault-token', `hvs.${v}\n`, 'credential-file'], + ]; + + it.each(FINDINGS)('reports %s: %j', (file, text, rule) => { + expect(rules(file, text)).toContain(rule); + for (const finding of scanText(file, text)) expect(Object.keys(finding).sort()).toEqual(['line', 'path', 'rule']); + }); + + // Placeholders, references and non-secret content in the same formats must stay quiet. + const CLEAN = [ + ['.netrc', 'machine api.internal login app password \n'], + ['.netrc', 'machine api.internal login app password ${NETRC_PASSWORD}\n'], + ['.netrc', `# password ${v}\nmachine api.internal login app\n`], + ['.netrc', 'machine api.internal login app\n'], + ['.pgpass', 'hostname:port:database:username:password\n'], + ['.pgpass', `# db.internal:5432:prod:app:${v}\n`], + ['.pgpass', 'db.internal:5432:prod:app:your_password_here\n'], + ['.git-credentials', 'https://user:password@github.com\n'], + ['.git-credentials', 'https://@github.com\n'], + ['.git-credentials', 'https://${GITHUB_TOKEN}@github.com\n'], + ['.npmrc', '//registry.npmjs.org/:_authToken=${NPM_TOKEN}\n'], + ['.npmrc', 'registry=https://registry.npmjs.org/\nalways-auth=true\nsave-exact=true\n'], + ['.npmrc', '_auth=\n'], + ['.yarnrc', '"//registry.npmjs.org/:_authToken" "${NPM_TOKEN}"\n'], + ['.yarnrc.yml', 'npmAuthToken: ${NPM_TOKEN}\nnpmRegistryServer: "https://registry.npmjs.org"\n'], + ['.pypirc', '[pypi]\nusername = __token__\npassword = \n'], + ['.pypirc', '[pypi]\nusername = __token__\npassword = ${PYPI_TOKEN}\n'], + ['.aws/credentials', '[default]\naws_secret_access_key = YOUR_SECRET_ACCESS_KEY\naws_access_key_id = AKIAIOSFODNN7EXAMPLE\n'], + ['.aws/credentials', '[default]\nregion = us-east-1\noutput = json\n'], + ['.docker/config.json', '{"auths":{"ghcr.io":{"auth":""}},"credsStore":"desktop"}\n'], + ['manifests/pull-secret.yaml', 'data:\n .dockerconfigjson: \n'], + ['kubeconfig', 'users:\n- user:\n client-certificate-data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCg==\n exec:\n command: aws\n'], + ['kubeconfig', `clusters:\n- cluster:\n certificate-authority-data: ${v}${v}\n`], + ['.htpasswd', '# users\nuser:password\n'], + ['.htpasswd', ''], + ['.my.cnf', '[client]\nuser=root\nhost=127.0.0.1\n'], + ['.my.cnf', '[client]\npassword=${MYSQL_PWD}\n'], + ['.s3cfg', '[default]\nsecret_key = \nhost_base = s3.amazonaws.com\n'], + ['prod.tfvars', 'db_password = var.db_password\n'], + ['.terraformrc', 'plugin_cache_dir = "$HOME/.terraform.d/plugin-cache"\n'], + ['.curlrc', 'silent\nuser = "name:${CURL_PASSWORD}"\n'], + ['.vault-token', '\n'], + ['.vault-token', '# token\n'], + ]; + + it.each(CLEAN)('does not report %s: %j', (file, text) => { + expect(rules(file, text)).toEqual([]); + }); + + it('ordinary code that merely mentions a netrc line is judged by content only (a five-field string in a source file is not a .pgpass line)', () => { + expect(rules('src/netrc-helper.js', `const line = "db.internal:5432:prod:app:${v}";\n`)).toEqual([]); + }); + + it('classifies credential files by name and directory, and scans them as config files', () => { + const cases = { + '.netrc': 'netrc', '_netrc': 'netrc', 'a/b/.pgpass': 'pgpass', '.git-credentials': 'gitcred', '.npmrc': 'npmrc', + '.yarnrc.yml': 'npmrc', '.pypirc': 'pypirc', '.aws/credentials': 'awscreds', 'x/.aws/config': 'awscreds', + '.docker/config.json': 'docker', '.dockercfg': 'docker', '.kube/config': 'kube', 'a.kubeconfig': 'kube', + '.htpasswd': 'htpasswd', '.htdigest': 'htpasswd', '.my.cnf': 'mycnf', '.s3cfg': 's3cfg', '.boto': 's3cfg', + '.terraformrc': 'terraformrc', 'terraform.tfstate': 'terraformrc', '.curlrc': 'curlrc', '.wgetrc': 'wgetrc', + '.vault-token': 'vault', + }; + for (const [file, tag] of Object.entries(cases)) { + expect([...credentialFormats(file)], file).toContain(tag); + expect(fileMode(file), file).toBe('config'); + } + expect(credentialFormats('src/index.js').size).toBe(0); + expect(credentialFormats('docs/config.json').size).toBe(0); + }); + + it('a credential in a strict format is reported on the line that holds it, with nothing but path/line/rule', () => { + const findings = scanText('.npmrc', `registry=https://r.internal.org/\n//r.internal.org/:_authToken=${v}\n`); + expect(findings).toEqual([{ path: '.npmrc', line: 2, rule: 'secret-assignment' }]); + const report = formatReport(findings); + for (const piece of windows(v)) expect(report).not.toContain(piece); + }); +}); + +// --------------------------------------------------------------------------- +// Review round 7 (finding 3): URLs that are bearer credentials +// +// Class: a URL VALUE under a strong secret name, in env, YAML, JSON and Markdown. Siblings: signed-URL query +// parameters (sig, signature, X-Amz-Signature, token, access_token, key, secret, password, auth), a random query value +// under any name, a random path segment, a webhook path token, a token used as the URL user name, a fragment token, +// a percent-encoded value, a JSON-escaped URL; and the webhook shapes themselves (Slack, Discord, Teams, Power +// Automate, Zapier, IFTTT, Telegram) in any file. +// --------------------------------------------------------------------------- + +describe('review round 7 (3): URL-valued secrets and webhook URLs', () => { + const v = randomString(24, 221); + const hex = randomString(64, 222, HEX); + const N = (...parts) => parts.join(''); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + + const URL_FINDINGS = [ + [`API_TOKEN=https://download.internal/file?sig=${v}`, '.env'], + [`API_TOKEN=https://download.internal/file?Signature=${v}&Expires=1700000000`, '.env'], + [`API_TOKEN="https://bucket.s3.amazonaws.com/o?X-Amz-Signature=${hex}&X-Amz-Expires=300"`, '.env'], + [`API_TOKEN=https://download.internal/file?X-Amz-Security-Token=${v}`, '.env'], + [`API_TOKEN=https://download.internal/file?access_token=${v}`, '.env'], + [`API_SECRET=https://download.internal/file?apikey=${v}`, '.env'], + [`API_SECRET=https://download.internal/file?key=${v}`, '.env'], + [`API_SECRET=https://download.internal/file?password=${v}`, '.env'], + [`API_TOKEN=https://download.internal/file?token=abcd1234`, '.env'], // short credential parameter + [`API_TOKEN=https://download.internal/file?file=${v}${v}`, '.env'], // random value under an ordinary name + [`API_TOKEN=https://download.internal/dl#access_token=${v}`, '.env'], + [`API_TOKEN=https://download.internal/file?sig=${encodeURIComponent(`${v}+/=`)}`, '.env'], + [`API_TOKEN=https://${v}@github.com/o/r`, '.env'], + [`WEBHOOK_TOKEN=https://hooks.internal.net/services/${v}`, '.env'], + [`WEBHOOK_TOKEN=https://hooks.example.net/services/${v}`, '.env'], // "example" in the host does not hide it + [`WEBHOOK_SECRET=https://hooks.internal.net/${hex}`, '.env'], + [`api_token: https://download.internal/file?sig=${v}`, 'c.yaml'], + [`api_token: 'https://download.internal/file?sig=${v}'`, 'c.yaml'], + [`downloadToken: "https://download.internal/file?sig=${v}"`, 'c.yaml'], + [`{"downloadToken": "https://download.internal/file?sig=${v}"}`, 'c.json'], + [`{"webhookSecret": "https://download.internal\\/hook\\/${v}"}`, 'c.json'], + [`Set API_TOKEN=https://download.internal/file?sig=${v} in your shell`, 'README.md'], + [`const cfg = { apiToken: "https://download.internal/file?sig=${v}" };`, 'src/a.js'], + [`API_TOKEN=\${BASE}/file?sig=${v}`, '.env'], // expansion next to a signed literal + ]; + + it.each(URL_FINDINGS)('reports %s in %s', (text, file) => { + expect(rules(file, `${text}\n`)).toContain('secret-assignment'); + }); + + const URL_CLEAN = [ + ['TOKEN_ENDPOINT=https://auth.internal.net/oauth/token', '.env'], + ['TOKEN_URL=https://auth.internal.net/oauth/token', '.env'], + [`TOKEN_URL=https://auth.internal.net/oauth/token?grant=${v}`, '.env'], // weak names keep the endpoint exemption + [`AUTH_URL=https://auth.internal.net/authorize?client_id=${v}`, '.env'], + ['API_TOKEN=https://api.internal.net/v1/items?page=2&sort=name', '.env'], + ['API_TOKEN=https://api.internal.net/v1/oauth/token', '.env'], + ['API_TOKEN=https://api.internal.net/v1/x?token=', '.env'], + ['API_TOKEN=https://api.internal.net/v1/x?token=${API_TOKEN_VALUE}', '.env'], + ['API_TOKEN=https://api.internal.net/v1/x?token=', '.env'], + ['API_TOKEN=https://hooks.internal.net/services/{team}/{bot}/{token}', '.env'], + ['API_TOKEN=https://hooks.internal.net/services/:team/:token', '.env'], + ['API_TOKEN=https:///path', '.env'], + ['api_token: https://api.internal.net/v1/health', 'c.yaml'], + ['{"apiToken": "https://api.internal.net/v1/health"}', 'c.json'], + ['Set API_TOKEN=https://api.internal.net/v1/health in your shell', 'README.md'], + ['API_TOKEN=https://api.internal.net/downloads/annual-fish-yield-report-2024.pdf', '.env'], + ]; + + it.each(URL_CLEAN)('does not report %s in %s', (text, file) => { + expect(rules(file, `${text}\n`)).toEqual([]); + }); + + it('a URL user name that is a template or a word is not a token', () => { + expect(rules('.env', 'API_TOKEN=https://${GITHUB_TOKEN}@github.com/o/r\n')).toEqual([]); + expect(rules('.env', 'API_TOKEN=https://deploy-bot@github.com/o/r\n')).toEqual([]); + }); + + const name = (host, tail) => `https://${host}/${tail}`; + const WEBHOOKS = [ + ['Slack service', name('hooks.slack.com', `services/T0123ABCD/B0123ABCD/${v}`)], + ['Slack workflow', name('hooks.slack.com', `workflows/T0123ABCD/A0123ABCD/123456789012/${v}`)], + ['Slack trigger', name('hooks.slack.com', `triggers/E0123ABCD/123456789012/${hex.slice(0, 32)}`)], + ['Slack, JSON-escaped slashes', name('hooks.slack.com', `services/T0123ABCD/B0123ABCD/${v}`).replaceAll('/', '\\/')], + ['Discord', name('discord.com', `api/webhooks/123456789012345678/${v}${v}`)], + ['Discord (discordapp.com, versioned)', name('discordapp.com', `api/v10/webhooks/123456789012345678/${v}${v}`)], + [ + 'Microsoft Teams', + name('contoso.webhook.office.com', `webhookb2/${hex.slice(0, 8)}-1111-2222-3333-444444444444@${hex.slice(8, 16)}-1111-2222-3333-444444444444/IncomingWebhook/${hex.slice(0, 32)}/${hex.slice(16, 24)}-1111-2222-3333-444444444444`), + ], + [ + 'Microsoft Teams (outlook.office.com)', + name('outlook.office.com', `webhook/${hex.slice(0, 8)}-1111-2222-3333-444444444444@${hex.slice(8, 16)}-1111-2222-3333-444444444444/IncomingWebhook/${hex.slice(0, 32)}/${hex.slice(16, 24)}-1111-2222-3333-444444444444`), + ], + ['Power Automate', name('prod-12.westus.logic.azure.com:443', `workflows/${hex.slice(0, 32)}/triggers/manual/paths/invoke?api-version=2016-06-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=${v}${v}`)], + ['Zapier', name('hooks.zapier.com', `hooks/catch/123456/${randomString(7, 223, LOWER_ALNUM)}/`)], + ['IFTTT', name('maker.ifttt.com', `trigger/fish_alert/with/key/${v}`)], + ['Telegram bot', name('api.telegram.org', `bot123456789:${v}${randomString(12, 224)}/sendMessage`)], + ]; + + it.each(WEBHOOKS)('webhook-url flags a %s webhook wherever it appears', (_label, url) => { + for (const file of ['deploy.sh', 'README.md', 'src/notify.js', 'config/alerts.yaml', '.env', 'hooks.json']) { + expect(rules(file, `curl -X POST ${url}\n`), file).toContain('webhook-url'); + } + const report = formatReport(scanText('README.md', `curl -X POST ${url}\n`)); + for (const piece of windows(v)) expect(report).not.toContain(piece); + }); + + const WEBHOOK_CLEAN = [ + // Built at runtime: a fully literal Slack-shaped URL is blocked by GitHub push protection even as a placeholder. + ['https://hooks.slack.com/services/T00000000/B00000000/', 'X'.repeat(24)].join(''), + 'https://hooks.slack.com/services/YOUR/WEBHOOK/URL', + 'https://hooks.slack.com/services/', + 'https://hooks.slack.com/services/${SLACK_TOKEN}', + 'https://discord.com/api/webhooks/123456789012345678/', + 'https://discord.com/api/webhooks/123456789012345678/REDACTED-REDACTED-REDACTED', + 'https://hooks.zapier.com/hooks/catch/123456/xxxxxx/', + 'https://maker.ifttt.com/trigger/event/with/key/your-ifttt-key-goes-here', + 'https://api.telegram.org/bot/sendMessage', + 'https://hooks.slack.com/', + 'https://api.slack.com/messaging/webhooks', + ]; + + it.each(WEBHOOK_CLEAN)('webhook-url leaves the placeholder or documentation link %s alone', (url) => { + expect(rules('README.md', `POST to ${url}\n`)).toEqual([]); + expect(rules('deploy.sh', `curl -X POST ${url}\n`)).toEqual([]); + }); + + it('a strong name holding a Slack webhook is reported by both the name and the URL', () => { + const url = name('hooks.slack.com', `services/T0123ABCD/B0123ABCD/${v}`); + expect(rules('.env', `${N('SLACK_', 'WEBHOOK_TOKEN')}=${url}\n`).sort()).toEqual(['secret-assignment', 'webhook-url']); + expect(rules('.env', `${N('SLACK_', 'WEBHOOK_URL')}=${url}\n`)).toEqual(['webhook-url']); + }); +}); + +// --------------------------------------------------------------------------- +// Review round 7 (finding 4): name and value fields in any order +// +// Class: a secret-like NAME field and a VALUE field in the same bounded object, with other fields in between and in +// either order: JSON (compact and pretty), YAML flow mappings, YAML block mappings and list items, Kubernetes env +// lists, Netlify-style nested values, Terraform blocks, JSON stored as an escaped string. Objects do not leak into +// their neighbours and the search never leaves a bounded window. +// --------------------------------------------------------------------------- + +describe('review round 7 (4): name/value pairs in any order', () => { + const v = randomString(24, 231); + const N = ['JWT_', 'SECRET'].join(''); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + const PAIR = 'secret-name-value-pair'; + + const FINDINGS = [ + ['JSON, value first', 'a.json', `[{"value":"${v}","key":"${N}"}]`], + ['JSON, field in between', 'a.json', `{"key":"${N}","type":"encrypted","value":"${v}"}`], + ['JSON, several fields in between', 'a.json', `{"key":"${N}","target":["production","preview"],"type":"encrypted","comment":"x","value":"${v}"}`], + ['JSON, value first with fields in between', 'a.json', `{"value":"${v}","type":"encrypted","target":["preview"],"name":"${N}"}`], + ['JSON, pretty printed', 'a.json', `[\n {\n "key": "${N}",\n "type": "encrypted",\n "target": ["production"],\n "value": "${v}"\n }\n]`], + ['JSON, pretty printed, value first', 'a.json', `[\n {\n "value": "${v}",\n "type": "encrypted",\n "name": "${N}"\n }\n]`], + ['JSON, second object of an array', 'a.json', `[{"key":"PORT","value":"3000"},{"value":"${v}","key":"${N}"}]`], + ['JSON, single quotes', 'a.json', `{'value':'${v}','key':'${N}'}`], + ['JSON, Netlify nested values', 'a.json', `{"key":"${N}","scopes":["builds"],"values":[{"value":"${v}","context":"all"}]}`], + ['JSON, nested values first', 'a.json', `{"values":[{"value":"${v}","context":"all"}],"key":"${N}"}`], + ['JSON, secretValue field', 'a.json', `{"secretValue":"${v}","name":"${N}"}`], + ['JSON stored as an escaped string', 'a.json', `{"data":"{\\"value\\":\\"${v}\\",\\"key\\":\\"${N}\\"}"}`], + ['Markdown code fence', 'a.md', `\`\`\`json\n{"value":"${v}","key":"${N}"}\n\`\`\``], + ['YAML flow mapping', 'a.yaml', `env: [{name: ${N}, value: ${v}}]`], + ['YAML flow mapping, value first', 'a.yaml', `env: [{value: ${v}, name: ${N}}]`], + ['YAML flow mapping, field in between', 'a.yaml', `env: [{name: ${N}, description: x, value: "${v}"}]`], + ['YAML list item, field in between', 'a.yaml', `env:\n - name: ${N}\n type: opaque\n value: ${v}`], + ['YAML list item, value first', 'a.yaml', `env:\n - value: ${v}\n type: opaque\n name: ${N}`], + ['YAML list item, value in the middle', 'a.yaml', `env:\n - type: opaque\n name: ${N}\n description: d\n value: "${v}"`], + ['YAML list item, name in the middle', 'a.yaml', `env:\n - type: opaque\n value: "${v}"\n name: ${N}\n scope: all`], + ['YAML plain mapping', 'a.yaml', `name: ${N}\ntype: opaque\nvalue: ${v}`], + ['YAML plain mapping, value first', 'a.yaml', `value: ${v}\ntype: opaque\nname: ${N}`], + ['Kubernetes env list', 'deploy.yaml', `containers:\n - name: app\n env:\n - name: PORT\n value: "3000"\n - name: ${N}\n value: ${v}`], + ['Kubernetes env list with a comment line', 'deploy.yaml', `env:\n - name: ${N}\n # rotate monthly\n value: ${v}`], + ['Terraform block', 'main.tf', `environment_variable {\n name = "${N}"\n type = "PLAINTEXT"\n value = "${v}"\n}`], + ['Terraform inline map, value first', 'main.tf', `env = { value = "${v}", name = "${N}" }`], + ['Docker Compose long-form list', 'docker-compose.yml', `services:\n app:\n environment:\n - name: ${N}\n value: ${v}`], + ]; + + it.each(FINDINGS)('reports: %s', (_label, file, text) => { + expect(rules(file, `${text}\n`)).toContain(PAIR); + const finding = scanText(file, `${text}\n`).find((f) => f.rule === PAIR); + expect(Object.keys(finding).sort()).toEqual(['line', 'path', 'rule']); + }); + + const CLEAN = [ + ['non-secret name', 'a.json', `{"value":"${v}","key":"PORT"}`], + ['placeholder value', 'a.json', `{"value":"","key":"${N}"}`], + ['environment reference', 'a.json', `{"key":"${N}","value":"\${${N}}"}`], + ['empty value', 'a.json', `{"key":"${N}","value":""}`], + ['short value', 'a.json', `{"key":"${N}","value":"dev"}`], + ['sentence value', 'a.json', `{"key":"reset_password_token","value":"Your session token has expired, please sign in again."}`], + ['value in the NEXT object of an array', 'a.json', `[{"key":"${N}","note":1},{"key":"OTHER","value":"${v}"}]`], + ['value in the PREVIOUS object of an array', 'a.json', `[{"key":"OTHER","value":"${v}"},{"key":"${N}","note":1}]`], + ['valueFrom instead of value', 'a.yaml', `env:\n - name: ${N}\n valueFrom:\n secretKeyRef:\n name: app-secrets\n key: jwt`], + ['secretKeyRef flow mapping', 'a.yaml', `env:\n - name: ${N}\n valueFrom:\n secretKeyRef: {name: app-secrets, key: jwt}`], + ['YAML: value belongs to the next item', 'a.yaml', `env:\n - name: ${N}\n note: 1\n - name: OTHER\n value: "${v}"`], + ['YAML: value belongs to the previous item', 'a.yaml', `env:\n - value: "${v}"\n name: OTHER\n - name: ${N}\n note: 1`], + ['YAML: value under a different document', 'a.yaml', `name: ${N}\n---\nvalue: ${v}`], + ['Terraform: reference value', 'main.tf', `environment_variable {\n name = "${N}"\n value = var.jwt_secret\n}`], + ['GitHub Actions secret reference', 'ci.yml', `steps:\n - name: deploy\n env:\n JWT_SECRET: \${{ secrets.JWT_SECRET }}`], + ['docker-compose interpolation', 'docker-compose.yml', `services:\n app:\n environment:\n - ${N}=\${${N}:?set it}\n - PORT=\${PORT:-3000}`], + ['value far outside the window', 'a.json', `{"key":"${N}","filler":"${'y'.repeat(2500)}","value":"${v}"}`], + ]; + + it.each(CLEAN)('does not report: %s', (_label, file, text) => { + expect(rules(file, `${text}\n`)).toEqual([]); + }); + + it('reports the line of the name field, and the allow marker on either line silences it', () => { + const text = `env:\n - value: ${v}\n name: ${N}\n`; + expect(scanText('a.yaml', text)).toEqual([{ path: 'a.yaml', line: 3, rule: PAIR }]); + expect(rules('a.yaml', `env:\n - value: ${v}\n name: ${N} # ${ALLOW_MARKER}\n`)).toEqual([]); + }); + + it('stays bounded: a very large object with the pair at both ends is not searched end to end', () => { + const filler = Array.from({ length: 400 }, (_, i) => `"f${i}":${i}`).join(','); + expect(rules('a.json', `{"key":"${N}",${filler},"value":"${v}"}\n`)).toEqual([]); + }); + + // The dense-file budget: every `name: SECRET` line costs about 250 characters of window, PAIR_BUDGET_CHARS is 24 million. + it('a file too dense to verify is reported exactly once, on the line where the budget ran out, instead of being searched forever', SLOW, () => { + const dense = ' name: SECRET\n'.repeat(150_000); + const started = performance.now(); + const findings = scanText('a.yaml', dense); + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + const pair = findings.filter((f) => f.rule === PAIR); + expect(pair).toHaveLength(1); + expect(pair[0].line).toBeGreaterThan(20_000); // not on the first lines: the budget has to run out first + expect(pair[0].line).toBeLessThanOrEqual(150_000); + }); + + it('a file just under the budget is searched in full and reports nothing', SLOW, () => { + expect(scanText('a.yaml', ' name: SECRET\n'.repeat(30_000)).filter((f) => f.rule === PAIR)).toEqual([]); + }); +}); + +describe('scan time', () => { + const repeat = (unit, bytes) => unit.repeat(Math.ceil(bytes / unit.length)); + const HUNDRED_KB = 100 * 1024; + const cases = [ + ['a keyword repeated in Markdown', 'notes.md', repeat('secret', HUNDRED_KB)], + ['token. repeated in Markdown', 'notes.md', repeat('token.', HUNDRED_KB)], + ['api_key repeated in an env file', '.env', repeat('api_key', HUNDRED_KB)], + ['name= repeated in an env file', '.env', repeat('secret=', HUNDRED_KB)], + ['weak name= chain (nested assignments)', '.env', repeat('secret_hint=', HUNDRED_KB)], + ['weak name= chain in Markdown', 'a.md', repeat('token_url=', HUNDRED_KB)], + ['a=b= chain', 'a.js', repeat('a=', HUNDRED_KB)], + ['process.env.SECRET repeated in code', 'a.js', repeat('process.env.SECRET', HUNDRED_KB)], + ['process.env.SECRET_TOKEN_ repeated in code', 'a.js', repeat('process.env.SECRET_TOKEN_', HUNDRED_KB)], + ['one very long env name', 'a.js', `process.env.${'SECRET'.repeat(HUNDRED_KB / 6)}`], + ['unterminated quoted values', '.env', repeat('password="x ', HUNDRED_KB)], + ['colon after colon', 'a.yaml', repeat('password: :', HUNDRED_KB)], + ['YAML name lines', 'a.yaml', repeat('- name: SECRET\n ', HUNDRED_KB)], + ['brace runs', 'a.js', repeat('{a', HUNDRED_KB)], + ['destructuring targets', 'a.js', repeat('{a} = process.env ', HUNDRED_KB)], + ['jwt.sign( repeated', 'a.js', repeat('jwt.sign(', HUNDRED_KB)], + ['SQL keywords', 'a.sql', repeat('alter role with password ', HUNDRED_KB)], + ['curl -u repeated', 'a.md', repeat(`curl -u a:${'b'} `, HUNDRED_KB)], + ['scheme-like runs', 'a.md', repeat('a.b+c-', HUNDRED_KB)], + ['PEM headers', 'a.md', repeat('-----BEGIN A PRIVATE KEY-----\n', HUNDRED_KB)], + ['{"key":"SECRET" objects, never closed', 'a.json', repeat('{"key":"SECRET"', HUNDRED_KB)], + ['"key":"SECRET" fields with no object', 'a.md', repeat('"key":"SECRET",', HUNDRED_KB)], + ['nested braces before name fields', 'a.json', `${'{'.repeat(5000)}${repeat('"key":"SECRET",', HUNDRED_KB)}`], + ['YAML name lines with siblings', 'a.yaml', repeat(' - name: SECRET\n type: x\n', HUNDRED_KB)], + ['YAML name lines at one indent', 'a.yaml', repeat(' name: SECRET\n', HUNDRED_KB)], + ['password repeated in a .netrc', '.netrc', repeat('password ', HUNDRED_KB)], + ['password + newline repeated in a .netrc', '.netrc', repeat('password\n', HUNDRED_KB)], + ['colons in a .pgpass', '.pgpass', repeat(':', HUNDRED_KB)], + ['five-field lines in a .pgpass', '.pgpass', repeat('a:b:c:d:e\n', HUNDRED_KB)], + ['user: repeated in a .htpasswd', '.htpasswd', repeat('a:', HUNDRED_KB)], + ['_auth repeated in a .npmrc', '.npmrc', repeat('_auth ', HUNDRED_KB)], + ['URL user@ runs in a .git-credentials', '.git-credentials', repeat('a://b@', HUNDRED_KB)], + ['signed-URL assignments', '.env', repeat('API_TOKEN=https://a/b?sig=', HUNDRED_KB)], + ['one URL with a huge query', '.env', `API_TOKEN=https://a/?${repeat('a=b&', HUNDRED_KB)}`], + ['webhook hosts repeated', 'a.md', repeat('hooks.slack.com/services/A/', HUNDRED_KB)], + ['Teams and Power Automate hosts repeated', 'a.md', repeat('.webhook.office.com/webhook/.logic.azure.com/workflows/', HUNDRED_KB)], + ['client-key-data: repeated', 'a.yaml', repeat('client-key-data: ', HUNDRED_KB)], + // Review round 8 (4): one-line credential files, where per-match line lookups used to make the scan quadratic. + ['"password x " repeated in a .netrc', '.netrc', repeat('password x ', HUNDRED_KB)], + ['"token: x " repeated in a kubeconfig', 'kubeconfig', repeat('token: x ', HUNDRED_KB)], + ['"a:b " repeated in a .htpasswd', '.htpasswd', repeat('a:b ', HUNDRED_KB)], + ['"user = a:x " repeated in a .curlrc', '.curlrc', repeat('user = a:x ', HUNDRED_KB)], + ['"auth": repeated in a docker config', 'config.json', repeat('"auth": "x" ', HUNDRED_KB)], + ['netrc content matcher on a README', 'a.md', repeat('machine a login b password ', HUNDRED_KB)], + ['pgpass content matcher on a script', 'a.sh', repeat('db:5432:d:u:x\n', HUNDRED_KB)], + ['"auths" and "auth" keys with no object', 'a.json', repeat('"auths":"auth":', HUNDRED_KB)], + ['{"key":"SECRET" objects in source code', 'a.js', repeat('{"key":"SECRET",', HUNDRED_KB)], + ['name( calls in source code', 'a.py', repeat('create(name="SECRET",', HUNDRED_KB)], + ['unclosed parentheses before name fields', 'a.js', repeat('((((name:"SECRET"', HUNDRED_KB)], + ['nested secret keys with children', 'a.yaml', repeat(' secret:\n x: 1\n', HUNDRED_KB)], + ['secret keys with an empty child block', 'a.yaml', repeat('password:\n ', HUNDRED_KB)], + ['XML entries with a name element', 'a.xml', repeat('SECRET', HUNDRED_KB)], + ['XML add entries', 'a.xml', repeat(' { + const started = performance.now(); + scanText(file, text); + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); + + // The 100 KB table cannot tell a quadratic scan from a linear one on a fast machine. These are a megabyte or more + // (a tracked file may be 5 MB): quadratic per-match line lookups took 19 to 23 seconds here, a linear scan takes + // well under half a second, and the limit is the same generous 8 s (about 20 times the linear time). + const ONE_MB = 1024 * 1024; + const BIG = [ + ['.netrc', repeat('password x ', ONE_MB)], + ['.netrc', repeat('password ', ONE_MB)], + ['kubeconfig', repeat('token: x ', 2 * ONE_MB)], + ['.htpasswd', repeat('a:b ', 2 * ONE_MB)], + ['.curlrc', repeat('user = a:x ', 2 * ONE_MB)], + ['.git-credentials', repeat('a://b@', 2 * ONE_MB)], + ['config.json', repeat('"auth": "x" ', ONE_MB)], + ['.vault-token', repeat('a ', 2 * ONE_MB)], + ]; + it.each(BIG)('a one-line %s of a megabyte or more scans in linear time', SLOW, (file, text) => { + const started = performance.now(); + scanText(file, text); + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); +}); + +// --------------------------------------------------------------------------- +// Review round 8: siblings of the credential-file, name/value pair, sentence and URL classes +// +// (1) Credential-file content under any name, and written by a script. Siblings: name variants (.bak, .prod, .txt, dot-, +// prefix and suffix), Markdown/text notes, echo/printf/heredoc writers in shell and CI YAML, JS and Dockerfile writers, +// docker config as compact/pretty JSON and YAML with auth/identitytoken/registrytoken, pgpass with wildcards. +// (2) Name/value pairs outside config files: JS/TS/Go objects, Python calls, CloudFormation, XML attribute and element +// forms, value-field synonyms, nested keys, YAML tags/anchors, CSV/TSV/Markdown rows, CLI invocations, and a brace or +// escaped quote inside a string field. +// (3) Sentences: what passes (message catalogs, documentation about a credential) and what does not (function words). +// (4) Scan time: see the 'scan time' tables above. +// --------------------------------------------------------------------------- + +describe('review round 8', () => { + const V = randomString(22, 801); + const N = ['JWT_', 'SECRET'].join(''); + const PASS = ['PASS', 'WORD'].join(''); + const B64 = Buffer.from(`u:${V}`).toString('base64'); + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + const PAIR = 'secret-name-value-pair'; + const CRED = 'credential-file'; + const CLI = 'secret-cli-command'; + + describe('(1) credential-file content under any name, and written by a script', () => { + const NETRC_LINE = `machine a.b login x password ${V}\n`; + it.each(['netrc.txt', 'config/.netrc.prod', '.netrc.bak', 'dot-netrc', '.netrc.local', 'netrc_backup', 'home/_netrc.old', 'NETRC'])( + 'a netrc line in %s is a finding', + (file) => { + expect(rules(file, NETRC_LINE)).toContain(CRED); + }, + ); + + it.each(['pgpass.txt', '.pgpass.bak', 'pgpass.local', 'db/pgpass-prod'])('a pgpass line in %s is a finding', (file) => { + expect(rules(file, `db:5432:d:u:${V}\n`)).toContain(CRED); + expect(rules(file, `*:*:*:app:${V}\n`)).toContain(CRED); + }); + + it.each(['git-credentials.txt', '.git-credentials.bak'])('a stored credential URL in %s is a finding', (file) => { + expect(rules(file, `https://${V}@github.com\n`)).toContain(CRED); + }); + + const DOCKER_JSON = `{"auths":{"h":{"auth":"${B64}"}}}`; + const DOCKER_PRETTY = `{\n "auths": {\n "h": {\n "auth": "${B64}"\n }\n }\n}\n`; + it.each([ + ['config.json', DOCKER_JSON], + ['docker-config.json', DOCKER_JSON], + ['ci/registry.json', DOCKER_JSON], + ['x.json', DOCKER_PRETTY], + ['x.yaml', `auths:\n h:\n auth: ${B64}\n`], + ['x.yaml', `auths:\n h:\n identitytoken: ${V}\n`], + ['x.json', `{"auths":{"h":{"registrytoken":"${V}"}}}`], + ['dockerconfigjson.bak', DOCKER_JSON], + ['Dockerfile', `RUN echo '{"auths":{"h":{"auth":"${B64}"}}}' > ~/.docker/config.json\n`], + ['deploy.js', `fs.writeFileSync(target, '{"auths":{"h":{"auth":"${B64}"}}}');\n`], + ['deploy.sh', `echo "{\\"auths\\":{\\"h\\":{\\"auth\\":\\"${B64}\\"}}}" > ~/.docker/config.json\n`], + ])('docker registry auth in %s (%#) is a finding', (file, text) => { + // credential-file (content), or secret-assignment when the file name already says "docker config". + expect(rules(file, text).some((rule) => rule === CRED || rule === 'secret-assignment')).toBe(true); + }); + + it.each([ + ['README.md', NETRC_LINE], + ['notes.txt', NETRC_LINE], + ['docs/guide.md', `machine a.b\n login x\n password ${V}\n`], + ['deploy.sh', `echo "machine github.com login x password ${V}" > ~/.netrc\n`], + ['.github/workflows/ci.yml', ` - run: echo "machine github.com login x password ${V}" > ~/.netrc\n`], + ['deploy.sh', `cat > ~/.netrc < ~/.netrc\n`], + ['deploy.sh', `echo "default login u password ${V}" >> ~/.netrc\n`], + ['deploy.sh', `echo "db:5432:d:u:${V}" > ~/.pgpass\n`], + ['.github/workflows/ci.yml', ` - run: echo "db:5432:d:u:${V}" > ~/.pgpass\n`], + ['deploy.sh', `cat > ~/.pgpass < { + expect(rules(file, text)).toContain(CRED); + }); + + it.each([ + ['README.md', 'machine learning password reset flow\n'], + ['a.txt', 'machine a.b login x password \n'], + ['a.txt', 'machine a.b login x password changeme\n'], + ['a.txt', `machine a.b login x password $${PASS}\n`], + ['a.sh', 'echo "machine github.com login x password xxxxxxxx" > ~/.netrc\n'], + ['a.md', 'machine ftp.example.com login anonymous password guest@\n'], + ['.netrc', 'machine ftp.example.com login anonymous password guest@\n'], + ['.netrc', 'machine ftp.example.com login anonymous password anonymous\n'], + ['a.txt', 'timestamps 2024:01:01:12:30:45 and 10:00:00:00:00\n'], + ['a.txt', 'fe80:0000:0000:0000:0001\n'], + ['a.sh', 'echo "db:5432:d:u:your_password" > ~/.pgpass\n'], + ['x.json', '{"auths":{"h":{"auth":""}}}'], + ['x.json', '{"auths":{}}'], + ['x.json', '{"auths":{"h":{"auth":""}}}'], + ['x.json', ['{"provider":{"', 'auth', '":"basic-auth-scheme-name-only"}}'].join('')], + ['x.yaml', ['oauth:\n ', 'auth', ': bearer-scheme-in-some-other-section\n'].join('')], + ['x.json', `{"auths":{},"filler":"${'y'.repeat(600)}","provider":{"${['au', 'th'].join('')}":"basic-scheme-name"}}`], // "auths" far above + ])('does not report %s: %j', (file, text) => { + expect(rules(file, text)).toEqual([]); + }); + + it('.htpasswd: a hash is a credential, a placeholder hash is documentation', () => { + expect(rules('.htpasswd', `user:$apr1$${V.slice(0, 8)}$${V}${V.slice(0, 4)}\n`)).toContain(CRED); + expect(rules('.htpasswd', 'user:$apr1$xxxxxxxx$xxxxxxxxxxxxxxxxxxxxxx\n')).toEqual([]); + expect(rules('.htpasswd', 'user:{SHA}REDACTED\n')).toEqual([]); + expect(rules('.htpasswd', `user:{SHA}${V}==\n`)).toContain(CRED); + }); + + it('reports path, line and rule only', () => { + const [finding] = scanText('deploy.sh', `echo hi\necho "machine github.com login x password ${V}" > ~/.netrc\n`); + expect(finding).toEqual({ path: 'deploy.sh', line: 2, rule: CRED }); + }); + + it('classifies the variant names as credential files', () => { + for (const [file, tag] of [ + ['netrc.txt', 'netrc'], ['.netrc.bak', 'netrc'], ['dot-netrc', 'netrc'], ['pgpass.txt', 'pgpass'], + ['git-credentials.txt', 'gitcred'], ['a/.docker/anything.json', 'docker'], ['docker-config.json', 'docker'], + ]) { + expect(credentialFormats(file).has(tag), file).toBe(true); + } + expect(credentialFormats('src/netrc-helper.js').has('netrc')).toBe(true); // by name; its content decides + expect(credentialFormats('netrcs.txt').has('netrc')).toBe(false); + }); + }); + + describe('(2) name/value pairs in code, XML, CloudFormation, CSV, Markdown and CLI invocations', () => { + const FINDINGS = [ + ['JS object, type field between', 'a.js', `await api.post('/env',{key:'${N}',type:'encrypted',value:'${V}'})`], + ['JS object, value first', 'a.js', `const x = {"value":"${V}","key":"${N}"};`], + ['TS array of objects', 'a.ts', `const x = [{name:"${N}",value:"${V}"}]`], + ['Go struct literal', 'a.go', `Env{Name:"${N}",Value:"${V}"}`], + ['Python call with keyword arguments', 'a.py', `create_var(name='${N}', value='${V}')`], + ['Python call, value first', 'a.py', `create_var(value='${V}', name='${N}')`], + ['Python dict', 'a.py', `x = {'name': '${N}', 'value': '${V}'}`], + ['Python call across lines', 'a.py', `create_var(\n name='${N}',\n description='x',\n value='${V}',\n)`], + ['CloudFormation YAML', 'a.yaml', `- ParameterKey: ${N}\n ParameterValue: ${V}`], + ['CloudFormation JSON', 'a.json', `[{"ParameterKey":"${N}","ParameterValue":"${V}"}]`], + ['Elastic Beanstalk option', 'a.json', `{"OptionName":"${N}","Value":"${V}"}`], + ['XML add, key then value', 'a.xml', ``], + ['XML add in a .config file', 'web.config', `\n \n`], + ['XML add, value then key', 'a.xml', ``], + ['XML setting with a value element', 'a.xml', `${V}`], + ['XML property with name and value elements', 'a.xml', `${N}${V}`], + ['XML property, pretty printed', 'a.xml', `\n ${N}\n ${V}\n`], + ['JSON content field', 'a.json', `{"name":"${N}","content":"${V}"}`], + ['JSON data field', 'a.json', `{"name":"${N}","data":"${V}"}`], + ['JSON default field', 'a.json', `{"name":"${N}","default":"${V}"}`], + ['JSON defaultValue field', 'a.json', `{"name":"${N}","defaultValue":"${V}"}`], + ['JSON stringValue field', 'a.json', `{"name":"${N}","stringValue":"${V}"}`], + ['nested YAML key with a value child', 'a.yaml', `secrets:\n ${N}:\n value: ${V}`], + ['nested YAML key, child after another field', 'a.yaml', `secrets:\n ${N}:\n type: opaque\n value: ${V}`], + ['nested JSON key with a value child', 'a.json', `{"${N}":{"value":"${V}"}}`], + ['YAML flow child', 'a.yaml', `${N}: {value: ${V}}`], + ['YAML !!str tag', 'a.yaml', `name: ${N}\nvalue: !!str ${V}`], + ['YAML anchor', 'a.yaml', `name: ${N}\nvalue: &a ${V}`], + ['YAML tag on a plain assignment', 'a.yaml', `${N}: !!str ${V}`, 'secret-assignment'], + ['YAML anchor on a plain assignment', 'a.yaml', `${N}: &secret ${V}`, 'secret-assignment'], + ['CSV row', 'a.csv', `${N},${V}`], + ['CSV row, quoted', 'a.csv', `"${N}","${V}"`], + ['TSV row', 'a.tsv', `${N}\t${V}`], + ['Markdown table row', 'a.md', `| ${N} | ${V} |`], + ['Markdown table row with code spans', 'a.md', `| \`${N}\` | \`${V}\` |`], + ['brace inside a string field', 'a.json', `{"name":"${N}","desc":"a } b","value":"${V}"}`], + ['brace inside a string field before the name', 'a.json', `{"value":"${V}","desc":"}","name":"${N}"}`], + ['escaped quote and brace in a string field', 'a.json', `{"name":"${N}","desc":"a\\"}\\"","value":"${V}"}`], + ['open brace inside a string field', 'a.json', `{"desc":"{","name":"${N}","value":"${V}"}`], + ['parenthesis inside a string argument', 'a.py', `create_var(name='${N}', note=')', value='${V}')`], + ]; + it.each(FINDINGS)('reports: %s', (_label, file, text, rule = PAIR) => { + expect(rules(file, `${text}\n`)).toContain(rule); + }); + + // The CLI shapes have their own rule. + const CLI_FINDINGS = [ + ['gh secret set --body', 'a.sh', `gh secret set ${N} --body ${V}`], + ['gh secret set --body in Markdown, quoted', 'a.md', `gh secret set ${N} --body "${V}"`], + ['gh secret set -b=', 'a.sh', `gh secret set ${N} -b${''}=${V}`], + ['gh variable set with --repo', 'a.sh', `gh secret set ${N} --repo o/r --body ${V}`], + ['vercel env add with a here-string', 'a.sh', `vercel env add ${N} production <<< ${V}`], + ['vercel env add fed by echo', 'a.sh', `echo ${V} | vercel env add ${N} production`], + ['vercel env add fed by printf %s', 'a.sh', `printf %s "${V}" | vercel env add ${N} production`], + ['netlify env:set', 'a.sh', `netlify env:set ${N} ${V}`], + ['aws ssm put-parameter', 'a.sh', `aws ssm put-parameter --name ${N} --value ${V} --type SecureString`], + ['aws ssm put-parameter, value first', 'a.sh', `aws ssm put-parameter --value ${V} --name /app/${N.toLowerCase()}`.replace('/app/', '')], + ['aws ssm inside a JS string', 'a.js', `exec("aws ssm put-parameter --name ${N} --value ${V}");`], + ['kubectl create secret --from-literal', 'a.sh', `kubectl create secret generic s --from-literal=${N}=${V}`], + ['heroku config:set in code', 'a.js', `exec('heroku config:set ${N}=${V}');`], + ['az keyvault secret set', 'a.sh', `az keyvault secret set --vault-name kv --name ${N} --value ${V}`], + ]; + it.each(CLI_FINDINGS)('reports the command: %s', (_label, file, text) => { + expect(rules(file, `${text}\n`)).toContain(CLI); + }); + + const CLEAN = [ + ['a placeholder value', 'a.json', `{"name":"${N}","value":"your_secret_here"}`], + ['a value that is an environment reference', 'a.js', `const x = {key:'${N}',value:process.env.${N}};`], + ['a bare identifier name in a function body', 'a.js', `function f() {\n const key = API_TOKEN;\n return { value: '${V}' };\n}`], + ['unrelated objects in one call', 'a.py', `f({name:'${N}'}, {name:'x', value:'${V}'})`], + ['a different entry after the name entry', 'a.xml', `\n`], + ['a CSV placeholder', 'a.csv', `${N},changeme`], + ['a Markdown table cell that is a description', 'a.md', `| ${N} | Signing secret for sessions |`], + ['a Markdown table label', 'a.md', `| ${N} | string |`], + ['a nested key with no value field', 'a.yaml', `secrets:\n ${N}:\n file: ./secret.txt`], + ['a nested key whose child is a sibling key', 'a.yaml', `${N}:\nvalue: ${V}`], + ['a value field that is another name entirely', 'a.json', `{"name":"${N}","valueFrom":"${V}"}`], + ['a YAML tag on a placeholder', 'a.yaml', `name: ${N}\nvalue: !!str changeme`], + ]; + it.each(CLEAN)('does not report: %s', (_label, file, text) => { + expect(rules(file, `${text}\n`)).toEqual([]); + }); + + const CLI_CLEAN = [ + ['an environment reference', `gh secret set ${N} --body "$${N}"`], + ['a braced environment reference', `gh secret set ${N} --body "\${${N}}"`], + ['a command substitution', `gh secret set ${N} --body "$(cat secret.txt)"`], + ['stdin from a file', `gh secret set ${N} < secret.txt`], + ['a placeholder', `netlify env:set ${N} your_token_here`], + ['a non-secret name', `gh secret set APP_MODE --body ${V}`], + ['a piped file, not a literal', `cat token.txt | vercel env add ${N} production`], + ['a comment after the command', `gh secret set ${N} # ${V}`], + ]; + it.each(CLI_CLEAN)('does not report the command with %s', (_label, text) => { + expect(rules('a.sh', `${text}\n`)).toEqual([]); + }); + + it('a finding carries path, line and rule only, on the name line', () => { + const text = `x = 1\nsecrets:\n ${N}:\n value: ${V}\n`; + expect(scanText('a.yaml', text)).toEqual([{ path: 'a.yaml', line: 3, rule: PAIR }]); + }); + + it('the allow marker silences an XML pair and a CLI command', () => { + expect(rules('a.xml', ` \n`)).toEqual([]); + expect(rules('a.sh', `gh secret set ${N} --body ${V} # ${ALLOW_MARKER}\n`)).toEqual([]); + }); + }); + + describe('(3) passphrase versus sentence, by context', () => { + // Function-word sentences a person may well have picked as a passphrase. + const PASSPHRASE_SENTENCES = [ + 'the horse is 123', + 'This is the way.', + 'My voice is my password.', + 'My name is Bond, James Bond.', + 'May the force be with you.', + 'One ring to rule them all.', + 'It was the best of times, it was the worst of times.', + 'it is what it is', + 'you shall not pass', + 'to be or not to be', + 'this is my super secret passphrase', + 'I have a dream today', + ]; + it.each(PASSPHRASE_SENTENCES)('%j is a finding under a strong name in .env, JSON, YAML and TOML', (phrase) => { + expect(rules('app/.env.x', `${N}="${phrase}"\n`)).toEqual(['secret-assignment']); + expect(rules('c.json', `{"${N}": "${phrase}"}\n`)).toEqual(['secret-assignment']); + expect(rules('c.yml', `${N.toLowerCase()}: "${phrase}"\n`)).toEqual(['secret-assignment']); + expect(rules('c.toml', `${N.toLowerCase()} = "${phrase}"\n`)).toEqual(['secret-assignment']); + }); + + // Documentation about the credential: instructions and references, not a value. + const DOCUMENTATION = [ + 'the password you chose during setup', + 'ask the team lead for the password', + 'the token from step 3', + 'whatever password you set in step 2', + 'same as the postgres password', + 'see the deployment guide for how to generate one', + 'a long random string of at least 32 characters', + 'The password for the database user', + 'Your session token has expired, please sign in again.', + 'see vault for the actual value', + 'set via environment variable at runtime', + ]; + it.each(DOCUMENTATION)('documentation %j is not a finding in Markdown, .env and YAML', (text) => { + expect(rules('README.md', `${PASS}="${text}"\n`)).toEqual([]); + expect(rules('.env.example', `${PASS}="${text}"\n`)).toEqual([]); + expect(rules('.env.example', `${PASS}=${text}\n`)).toEqual([]); + expect(rules('config.yml', `${PASS.toLowerCase()}: ${text}\n`)).toEqual([]); + }); + + it('documentation with a random-looking word in it is still a finding', () => { + expect(rules('README.md', `${PASS}="the password is ${V} see step 3"\n`)).toEqual(['secret-assignment']); + }); + + it('a number that counts nothing is part of a passphrase, a counted number is prose', () => { + expect(rules('locales/en.json', `{"${N}": "the horse is 123"}\n`)).toEqual(['secret-assignment']); + expect(rules('locales/en.json', `{"${N}": "Password must be at least 8 characters"}\n`)).toEqual([]); + expect(rules('locales/en.json', `{"${N}": "Wait 30 seconds and try again"}\n`)).toEqual([]); + expect(rules('locales/en.json', `{"${N}": "Enter the 6-digit code"}\n`)).toEqual([]); + }); + + it('UI sentences with the newly added vocabulary pass in a message catalog', () => { + for (const text of ['API key not found', 'Copy the token and paste it here', 'Password must contain at least one number and one symbol.']) { + expect(rules('locales/en.json', `{"resetToken": "${text}"}\n`), text).toEqual([]); + } + }); + + it('message catalog paths are recognised by directory and by file name', () => { + const phrase = 'It is what it is.'; + for (const file of ['i18n/x.json', 'src/locales/x.yml', 'lang/x.json', 'app/messages/x.yml', 'en.json', 'pt-BR.yml', 'messages_de.properties', 'translations.json', 'errors.json']) { + expect(rules(file, file.endsWith('.json') ? `{"${N}": "${phrase}"}\n` : `${N.toLowerCase()}: "${phrase}"\n`), file).toEqual([]); + } + for (const file of ['config.json', 'src/settings.yml', 'app/.env.x', 'enough.json', 'english.json']) { + expect(rules(file, file.endsWith('.json') ? `{"${N}": "${phrase}"}\n` : `${N.toLowerCase()}: "${phrase}"\n`), file).toEqual(['secret-assignment']); + } + }); + }); + + describe('(class) a URL that is itself a bearer credential, under a strong name', () => { + const TOKEN_NAME = ['API_', 'TOKEN'].join(''); + const FINDINGS = [ + ['signed URL in .env', 'a.env', `${TOKEN_NAME}=https://download.internal/file?sig=${V}`], + ['X-Amz-Signature in .env', 'a.env', `${TOKEN_NAME}=https://download.internal/file?X-Amz-Signature=${V}`], + ['access_token query in .env', 'a.env', `${TOKEN_NAME}=https://api.internal/v1/export?access_token=${V}`], + ['webhook path token in .env', 'a.env', `WEBHOOK_TOKEN=https://hooks.example.net/services/${V}`], + ['signed URL as a JSON value', 'a.json', `{"${TOKEN_NAME}":"https://download.internal/file?sig=${V}"}`], + ['signed URL as a YAML value', 'a.yaml', `${TOKEN_NAME.toLowerCase()}: https://download.internal/file?sig=${V}`], + ['signed URL as a quoted YAML value', 'a.yaml', `${TOKEN_NAME.toLowerCase()}: "https://download.internal/file?sig=${V}"`], + ['Slack webhook as a YAML value', 'a.yaml', `slack_webhook_secret: "https://hooks.slack.com/services/T0AAAAAAA/B0AAAAAAA/${V}"`], + ['Discord webhook under a strong name', 'a.env', `DISCORD_WEBHOOK_SECRET=https://discord.com/api/webhooks/123456789012/${V}${V}`], + ['token as the user name of a URL', 'a.env', `${TOKEN_NAME}=https://${V}@api.internal/v1`], + ]; + it.each(FINDINGS)('reports: %s', (_label, file, text) => { + expect(rules(file, `${text}\n`)).toContain('secret-assignment'); + }); + + it.each([ + ['Slack', `https://hooks.slack.com/services/T0AAAAAAA/B0AAAAAAA/${V}`], + ['Discord', `https://discord.com/api/webhooks/123456789012/${V}${V}`], + ['Zapier', `https://hooks.zapier.com/hooks/catch/123456/${V.slice(0, 10)}`], + ])('a bare %s webhook URL in Markdown is a finding of its own', (_label, url) => { + expect(rules('a.md', `${url}\n`)).toContain('webhook-url'); + }); + + it.each([ + ['a plain endpoint', `${TOKEN_NAME}=https://api.internal/v1/tokens`], + ['an endpoint with an id and a template', `${TOKEN_NAME}=https://api.internal/v1/users/{userId}/tokens`], + ['a weak name with a signed URL', `TOKEN_URL=https://download.internal/file?sig=${V}`], + ['a weak name, endpoint', 'TOKEN_ENDPOINT=https://auth.example.net/oauth/token'], + ['a page and a language', `${TOKEN_NAME}=https://docs.internal/guide?lang=en&page=2`], + ])('does not report %s', (_label, text) => { + expect(rules('a.env', `${text}\n`)).toEqual([]); + }); + }); + + describe('(low) webhook and signed-URL shapes', () => { + it('a PagerDuty integration URL under a strong name is a finding; a placeholder one is not', () => { + const key = randomString(32, 811, HEX); + expect(rules('a.env', `PAGERDUTY_TOKEN=https://events.pagerduty.com/integration/${key}/enqueue\n`)).toContain('secret-assignment'); + expect(rules('a.md', `curl https://events.pagerduty.com/integration/${key}/enqueue\n`)).toContain('webhook-url'); + expect(rules('a.md', 'curl https://events.pagerduty.com/integration//enqueue\n')).toEqual([]); + }); + + it('a percent-encoded character in a Slack webhook token does not hide it', () => { + expect(rules('a.env', `WEBHOOK_TOKEN=https://hooks.slack.com/services/T0AAAAAAA/B0AAAAAAA/%41${V}\n`)).not.toEqual([]); + expect(rules('a.md', `https://hooks.slack.com/services/T0AAAAAAA/B0AAAAAAA/%41${V}\n`)).toContain('webhook-url'); + }); + + it('weak names keep the endpoint exemption (a signed URL under TOKEN_URL is an address by design)', () => { + expect(rules('a.env', `TOKEN_URL=https://x.com/dl?X-Amz-Signature=${randomString(40, 812, HEX)}\n`)).toEqual([]); + expect(rules('a.env', 'TOKEN_ENDPOINT=https://auth.example.net/oauth/token\n')).toEqual([]); + }); + }); +}); + +// --------------------------------------------------------------------------- +// Skip list and decoding +// --------------------------------------------------------------------------- + +// A tracked file NAME is attacker-controlled (a pull request can add any name), and the scanner classifies every +// path with regexes. CodeQL (js/redos) found one whose separator and segment classes overlapped: 'i18n-' plus +// '--' x 22 took 10 to 16 s and doubled with every repeat. The content-based hostile-input tests above never +// exercised this because they vary the file TEXT, not the file PATH. +describe('hostile file paths are classified in linear time', () => { + // A backtracking regex blocks the JS thread, so Vitest's own timeout cannot interrupt it: if this bug ever came + // back, an in-process test would hang CI instead of failing. The hostile scans therefore run in a child process + // with a hard kill timeout, and the test fails cleanly (status null, signal SIGTERM) if the child is killed. + const CHILD_KILL_MS = 30_000; + const HOSTILE_PATH_LIMIT_MS = 2000; + const runHostile = (script) => + spawnSync(process.execPath, ['--input-type=module', '-e', script], { + cwd: REPO_ROOT, + encoding: 'utf8', + timeout: CHILD_KILL_MS, + }); + const SCAN_IMPORT = `import { scanText } from ${JSON.stringify(pathToFileURL(SCANNER).href)};`; + + it('the exact CodeQL shapes finish at once (each doubled per repeat before the fix)', SLOW, () => { + const result = runHostile(`${SCAN_IMPORT} + const body = 'MSG_TOKEN="This is the way."\\n'; + const makers = [ + (n) => 'i18n-' + '--'.repeat(n) + '!.json', + (n) => 'messages' + '__'.repeat(n) + '!.json', + (n) => 'errors' + '_-'.repeat(n) + '!.yaml', + ]; + const started = performance.now(); + for (const make of makers) { scanText(make(40), body); scanText(make(200), body); } + console.log(Math.round(performance.now() - started));`); + expect(result.error, 'the child was killed: a path regex is backtracking').toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + expect(Number(result.stdout.trim())).toBeLessThan(HOSTILE_PATH_LIMIT_MS); + }); + + it('no known path shape backtracks: 3 KB paths built from every name, repeat unit and ending', SLOW, () => { + const result = runHostile(`${SCAN_IMPORT} + const body = 'MSG_TOKEN="This is the way."\\n'; + const names = ['netrc', '.netrc', '_netrc', 'pgpass', '.pgpass', '.npmrc', '.yarnrc', '.pypirc', 'credentials', + 'config', 'docker-config', 'kubeconfig', 'htpasswd', '.htpasswd', 'my.cnf', '.s3cfg', 'id_rsa', 'messages', + 'errors', 'strings', 'en', 'locales', 'res/values', 'settings.xml', 'pom.xml', 'strings.xml', 'package-lock.json', 'yarn.lock', 'Cargo.lock', 'go.sum', 'i18n', 'l10n', 'translations', '.env', 'secrets', 'terraform', + '.git-credentials', '.curlrc', '.wgetrc', '.vault-token', 'docs/API']; + const units = ['-', '_', '.', '--', '__', '..', '-_', '_-', '.-', '-.', ' ', 'a-', '.a', 'a_', '/', '/.']; + let worst = 0; let where = ''; + for (const name of names) for (const unit of units) for (const tail of ['!', '.json', '/x.json']) { + const hostile = name + unit.repeat(Math.floor(3000 / unit.length)) + tail; + const started = performance.now(); scanText(hostile, body); const took = performance.now() - started; + if (took > worst) { worst = took; where = name + ' + ' + JSON.stringify(unit) + ' + ' + tail; } + } + console.log(JSON.stringify({ worst: Math.round(worst), where }));`); + expect(result.error, 'the child was killed: a path regex is backtracking').toBeUndefined(); + expect(result.status, result.stderr).toBe(0); + const { worst, where } = JSON.parse(result.stdout.trim()); + expect(worst, `slowest hostile path: ${where}`).toBeLessThan(HOSTILE_PATH_LIMIT_MS); + }); + + it('ordinary message-catalog file names are still recognised after the fix', () => { + const body = 'MSG_TOKEN="This is the way."\n'; + // A sentence under a token-like name is only exempt inside a message catalog. + for (const file of [ + 'src/messages.en.json', 'errors_en-US.json', 'strings-fr.xml', 'en.json', 'pt-BR.json', 'locales/de/app.json', + 'translations.es.yaml', 'i18n.zh-Hans.json', + ]) { + expect(scanText(file, body), file).toEqual([]); + } + // Same text in an ordinary config file is still a finding. + for (const file of ['config.json', 'app.env', 'settings.yaml']) { + expect(scanText(file, body).length, file).toBeGreaterThan(0); + } + }); +}); + +describe('isLockfile', () => { + it.each([ + 'package-lock.json', + 'app/package-lock.json', + 'server/package-lock.json', + 'npm-shrinkwrap.json', + 'yarn.lock', + 'pnpm-lock.yaml', + 'Cargo.lock', + 'Gemfile.lock', + 'poetry.lock', + 'composer.lock', + 'Pipfile.lock', + 'bun.lock', + 'go.sum', + ])('%s is a lockfile (scanned with the lockfile rules, not skipped)', (p) => { + expect(isLockfile(p)).toBe(true); + }); + + it.each([ + 'app/.env.production', + 'server/server.js', + 'scripts/check-secrets.mjs', + THIS_FILE_REL, + 'docs/API.md', + '.claude/settings.json', + '.claude/skills/tdd/SKILL.md', + 'app/package.json', + 'app/public/logo.png', + 'research/paper.pdf', + 'fonts/x.woff2', + 'secrets.lock', + 'notes/my.lock', + ])('%s is scanned with every rule (content decides, not name or extension)', (p) => { + expect(isLockfile(p)).toBe(false); + }); +}); + +describe('decodeText', () => { + const text = 'A=1\r\nAPI_KEY=abcdef\r\n'; + + it('reads UTF-8, with or without a BOM', () => { + expect(decodeText(Buffer.from(text))).toBe(text); + expect(decodeText(Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), Buffer.from(text)]))).toBe(text); + }); + + it('decodes UTF-16LE and UTF-16BE with a BOM instead of calling them binary', () => { + const le = Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(text, 'utf16le')]); + expect(decodeText(le)).toBe(text); + const be = Buffer.concat([Buffer.from([0xfe, 0xff]), Buffer.from(text, 'utf16le').swap16()]); + expect(decodeText(be)).toBe(text); + }); + + it('decodes BOM-less UTF-16 of ASCII text', () => { + expect(decodeText(Buffer.from(text, 'utf16le'))).toBe(text); + expect(decodeText(Buffer.from(text, 'utf16le').swap16())).toBe(text); + }); + + it('still calls real binary content binary', () => { + const png = Buffer.concat([Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 0x0d]), Buffer.alloc(200, 0)]); + expect(decodeText(png)).toBeNull(); + // ...but a NUL alone is not proof: without a known binary signature the bytes are decoded, NULs removed. + expect(decodeText(Buffer.from([1, 2, 0, 3, 0, 0, 9, 8, 0, 7, 0, 0, 1]))).toBe('\u0001\u0002\u0003\u0009\u0008\u0007\u0001'); + }); + + it('a stray NUL does not make a text file binary, wherever it sits', () => { + expect(decodeText(Buffer.from(`\0${text}`))).toBe(text); + expect(decodeText(Buffer.from(`${text}\0`))).toBe(text); + expect(decodeText(Buffer.from(`API_\0KEY=abcdef\n`))).toBe('API_KEY=abcdef\n'); + expect(isBinaryContent(Buffer.from(`\0${text}`))).toBe(false); + expect(isBinaryContent(PNG_HEAD)).toBe(true); + }); + + it('decodes UTF-32 with a BOM and without one (LE and BE)', () => { + const codepoints = [...text].map((c) => c.codePointAt(0)); + const le = Buffer.alloc(codepoints.length * 4); + const be = Buffer.alloc(codepoints.length * 4); + codepoints.forEach((cp, i) => { + le.writeUInt32LE(cp, i * 4); + be.writeUInt32BE(cp, i * 4); + }); + expect(decodeText(Buffer.concat([Buffer.from([0xff, 0xfe, 0, 0]), le]))).toBe(text); + expect(decodeText(Buffer.concat([Buffer.from([0, 0, 0xfe, 0xff]), be]))).toBe(text); + expect(decodeText(le)).toBe(text); + expect(decodeText(be)).toBe(text); + }); +}); + +// --------------------------------------------------------------------------- +// Self-check and the real repository +// --------------------------------------------------------------------------- + +describe('this repository', () => { + it('the scanner and its own test file scan clean', () => { + const scanner = readFileSync(SCANNER, 'utf8'); + const self = readFileSync(THIS_FILE, 'utf8'); + expect(scanText('scripts/check-secrets.mjs', scanner)).toEqual([]); + expect(scanText(THIS_FILE_REL, self)).toEqual([]); + }); + + const inGitRepo = spawnSync('git', ['rev-parse', '--show-toplevel'], { cwd: REPO_ROOT }).status === 0; + + it.skipIf(!inGitRepo)('a full-tree run exits 0 from the repo root', SLOW, () => { + const result = spawnSync(process.execPath, ['scripts/check-secrets.mjs'], { + cwd: REPO_ROOT, + encoding: 'utf8', + timeout: SLOW_TEST_MS, + }); + expect(result.stderr).toBe(''); + expect(result.stdout).toMatch(/^check-secrets: OK \(\d+ files scanned, \d+ skipped/); + expect(result.status).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// CLI behavior in throwaway git repositories +// --------------------------------------------------------------------------- + +describe('CLI', () => { + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + + const run = (cmd, args, cwd) => spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: SLOW_TEST_MS }); + const scan = (cwd, ...args) => run(process.execPath, [SCANNER, ...args], cwd); + const git = (cwd, ...args) => + run('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', '-c', 'commit.gpgsign=false', ...args], cwd); + const commit = (cwd, message) => git(cwd, 'commit', '-q', '-m', message); + + function makeRepo() { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-')); + dirs.push(dir); + expect(run('git', ['init', '-q'], dir).status).toBe(0); + return dir; + } + + const write = (dir, file, content) => { + const target = path.join(dir, file); + mkdirSync(path.dirname(target), { recursive: true }); + writeFileSync(target, content); + }; + + const secret = randomString(32, 61); + const assignment = `${['API_', 'KEY'].join('')}=${secret}\n`; + + // A text file just over the 5 MB limit, built cheaply: 1 KB lines (few lines keep git's diff and our parser fast), no + // per-line loop, `first` at the start. Never build these from millions of tiny lines. + const OVER_LIMIT_LINES = 5 * 1024 + 8; + const overLimitText = (first = '') => `${first}${`${'x'.repeat(1023)}\n`.repeat(OVER_LIMIT_LINES)}`; + + it.skipIf(!hasGit())('reports path, line and rule, exits 1, and leaks nothing', SLOW, () => { + const dir = makeRepo(); + write(dir, '.env', `FOO=1\n${assignment}`); + write(dir, 'README.md', 'nothing here\n'); + // Skipped on purpose: real binary content. A lockfile is scanned with the lockfile rules only, so a generic name=value + // (which would be an integrity-hash false positive) is not reported there. + write(dir, 'package-lock.json', assignment); + write(dir, 'blob.dat', Buffer.concat([PNG_HEAD, Buffer.from(assignment)])); + run('git', ['add', '-A'], dir); + + const result = scan(dir); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status).toBe(1); + expect(result.stderr).toContain('.env:2 secret-assignment'); + for (const piece of windows(secret)) expect(output).not.toContain(piece); + expect(output).not.toContain('package-lock.json'); + expect(output).not.toContain('blob.dat'); + expect(output).not.toContain('README.md'); + }); + + it.skipIf(!hasGit())('scans skill files, *.lock files and text files with a binary-looking extension', SLOW, () => { + const dir = makeRepo(); + const awsKey = (seed) => ['AKIA', randomString(16, seed, UPPER_ALNUM)].join(''); + write(dir, '.claude/skills/x/SKILL.md', `${awsKey(62)}\n`); + write(dir, 'deps.lock', `${awsKey(63)}\n`); + write(dir, 'logo.png', `${awsKey(64)}\n`); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('.claude/skills/x/SKILL.md:1 aws-access-key-id'); + expect(result.stderr).toContain('deps.lock:1 aws-access-key-id'); + expect(result.stderr).toContain('logo.png:1 aws-access-key-id'); + }); + + it.skipIf(!hasGit())('scans UTF-16 files (a Windows PowerShell redirect writes UTF-16LE with a BOM)', SLOW, () => { + const dir = makeRepo(); + writeFileSync(path.join(dir, 'win.env'), Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(`A=1\r\n${assignment}`, 'utf16le')])); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('win.env:2 secret-assignment'); + }); + + it.skipIf(!hasGit())('says how many files were skipped and why', SLOW, () => { + const dir = makeRepo(); + write(dir, 'package-lock.json', '{}\n'); + write(dir, 'blob.dat', Buffer.concat([PNG_HEAD, Buffer.from('binary\n')])); + write(dir, 'src/app.js', 'export const x = 1;\n'); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(0); + expect(result.stdout).toMatch(/2 files scanned, 1 skipped: /); + expect(result.stdout).not.toContain('lockfile'); // lockfiles are scanned now, not skipped + expect(result.stdout).toContain('1 binary'); + }); + + it.skipIf(!hasGit())('fails, and names the file, when a text file is too large to scan', SLOW, () => { + const dir = makeRepo(); + write(dir, 'data/big.json', overLimitText()); + write(dir, 'data/big.bin', Buffer.concat([PNG_HEAD, Buffer.alloc(5 * 1024 * 1024 + 4096, 0)])); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('NOT scanned'); + expect(result.stderr).toContain('data/big.json'); + expect(result.stderr).not.toContain('data/big.bin'); + }); + + it.skipIf(!hasGit())('exits 0 on a clean repository and scans from a subdirectory', SLOW, () => { + const dir = makeRepo(); + write(dir, '.env.example', 'API_KEY=your_api_key_here\n'); + write(dir, 'src/app.js', 'export const x = 1;\n'); + run('git', ['add', '-A'], dir); + expect(scan(dir).status).toBe(0); + expect(scan(path.join(dir, 'src')).status).toBe(0); + + write(dir, '.env', assignment); + run('git', ['add', '-A'], dir); + expect(scan(path.join(dir, 'src')).status).toBe(1); + }); + + it.skipIf(!hasGit())('does not scan untracked files (tracked files only)', SLOW, () => { + const dir = makeRepo(); + write(dir, 'tracked.txt', 'ok\n'); + run('git', ['add', '-A'], dir); + write(dir, '.env', assignment); + expect(scan(dir).status).toBe(0); + }); + + it.skipIf(!hasGit())('honors the inline allow marker', SLOW, () => { + const dir = makeRepo(); + write(dir, '.env', `${assignment.trimEnd()} # check-secrets:allow\n`); + run('git', ['add', '-A'], dir); + expect(scan(dir).status).toBe(0); + }); + + it.skipIf(!hasGit())('--history finds a removed secret, prints only commit/path/rule/count', SLOW, () => { + const dir = makeRepo(); + write(dir, '.env', assignment); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add config').status).toBe(0); + const leakingCommit = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + + write(dir, '.env', 'API_KEY=your_api_key_here\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'remove secret').status).toBe(0); + const cleanCommit = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + + // The current tree is clean... + expect(scan(dir).status).toBe(0); + // ...but history is not. + const result = scan(dir, '--history'); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${leakingCommit} .env secret-assignment x1`); + expect(result.stderr).not.toContain(cleanCommit); + for (const piece of windows(secret)) expect(output).not.toContain(piece); + }); + + it.skipIf(!hasGit())('--history works in a bare mirror clone', SLOW, () => { + const dir = makeRepo(); + write(dir, '.env', assignment); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add config').status).toBe(0); + const mirror = mkdtempSync(path.join(tmpdir(), 'check-secrets-mirror-')); + dirs.push(mirror); + expect(run('git', ['clone', '-q', '--mirror', dir, mirror], tmpdir()).status).toBe(0); + const result = scan(mirror, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain('.env secret-assignment x1'); + // Tree mode needs a work tree and says so instead of crashing. + expect(scan(mirror).status).toBe(2); + }); + + it.skipIf(!hasGit())('--history exits 0 when no commit ever added a secret, and in an empty repository', SLOW, () => { + const dir = makeRepo(); + expect(scan(dir, '--history').status).toBe(0); + write(dir, '.env.example', 'API_KEY=your_api_key_here\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add template').status).toBe(0); + const result = scan(dir, '--history'); + expect(result.status).toBe(0); + expect(result.stdout).toContain('no hits in 1 commits'); + }); + + it.skipIf(!hasGit())('--history fails (exit 2) instead of reporting "no hits" when git log itself fails', SLOW, () => { + const dir = makeRepo(); + write(dir, '.env.example', 'API_KEY=your_api_key_here\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add template').status).toBe(0); + const head = run('git', ['rev-parse', 'HEAD'], dir).stdout.trim(); + rmSync(path.join(dir, '.git', 'objects', head.slice(0, 2), head.slice(2)), { force: true }); + const result = scan(dir, '--history'); + expect(result.status).toBe(2); + expect(result.stdout).not.toContain('no hits'); + expect(result.stderr).toContain('git log failed'); + }); + + it.skipIf(!hasGit())('--history reads a line that follows a lone carriage return', SLOW, () => { + const dir = makeRepo(); + write(dir, 'lonecr.env', `A=1\r${assignment.replace(/\n/g, '\r')}`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'cr').status).toBe(0); + expect(scan(dir).stderr).toContain('lonecr.env:1 secret-assignment'); + expect(scan(dir, '--history').stderr).toContain('lonecr.env secret-assignment x1'); + }); + + it.skipIf(!hasGit())('--history reads UTF-16 files that git shows as binary', SLOW, () => { + const dir = makeRepo(); + writeFileSync(path.join(dir, 'win.env'), Buffer.concat([Buffer.from([0xff, 0xfe]), Buffer.from(`A=1\r\n${assignment}`, 'utf16le')])); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'utf16').status).toBe(0); + expect(scan(dir, '--history').stderr).toContain('win.env secret-assignment x1'); + }); + + it.skipIf(!hasGit())('--history scans a secret that only a merge conflict resolution introduced', SLOW, () => { + const dir = makeRepo(); + write(dir, '.env', 'A=1\nCONF=base\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'base').status).toBe(0); + const trunk = run('git', ['rev-parse', '--abbrev-ref', 'HEAD'], dir).stdout.trim(); + git(dir, 'checkout', '-q', '-b', 'feature'); + write(dir, '.env', 'A=1\nCONF=feature\n'); + expect(git(dir, 'commit', '-q', '-am', 'feature').status).toBe(0); + git(dir, 'checkout', '-q', trunk); + write(dir, '.env', 'A=1\nCONF=trunk\n'); + expect(git(dir, 'commit', '-q', '-am', 'trunk').status).toBe(0); + expect(git(dir, 'merge', 'feature').status).toBe(1); // conflict (git exits 1), resolved below + write(dir, '.env', `A=1\n${assignment}`); + git(dir, 'add', '-A'); + expect(commit(dir, 'merge').status).toBe(0); + const merge = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + + const result = scan(dir, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${merge} .env secret-assignment x1`); + // The parents' commits added nothing secret-like, so exactly one commit is reported. + expect(result.stderr).toContain('in 1 of 4 commits'); + for (const piece of windows(secret)) expect(`${result.stdout}${result.stderr}`).not.toContain(piece); + }); + + it.skipIf(!hasGit())('--history catches a value added on its own line under an unchanged secret-like name', SLOW, () => { + const dir = makeRepo(); + const name = ['JWT_', 'SECRET'].join(''); + const value = randomString(24, 74); + const manifest = (v) => `env:\n - name: ${name}\n value: ${v}\n - name: PORT\n value: "3000"\n`; + write(dir, 'deploy.yaml', manifest('changeme')); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add manifest').status).toBe(0); + write(dir, 'deploy.yaml', manifest(value)); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'leak').status).toBe(0); + const leakingCommit = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + write(dir, 'deploy.yaml', manifest('changeme')); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'remove').status).toBe(0); + const cleanCommit = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + + expect(scan(dir).status).toBe(0); + const result = scan(dir, '--history'); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${leakingCommit} deploy.yaml secret-name-value-pair x1`); + expect(result.stderr).toContain('in 1 of 3 commits'); + expect(result.stderr).not.toContain(cleanCommit); + for (const piece of windows(value)) expect(output).not.toContain(piece); + }); + + it.skipIf(!hasGit())('--history does not blame a later commit for an old value that only appears as context', SLOW, () => { + const dir = makeRepo(); + const name = ['JWT_', 'SECRET'].join(''); + const value = randomString(24, 75); + write(dir, 'deploy.yaml', `- name: ${name}\n value: ${value}\nport: 3000\n`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'leak').status).toBe(0); + const leakingCommit = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + // Only the line right under the pair changes; the pair stays as unchanged context inside the same hunk. + write(dir, 'deploy.yaml', `- name: ${name}\n value: ${value}\nport: 8080\n`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'change port').status).toBe(0); + const laterCommit = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + const result = scan(dir, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain(leakingCommit); + expect(result.stderr).not.toContain(laterCommit); + expect(result.stderr).toContain('in 1 of 2 commits'); + }); + + it.skipIf(!hasGit())('--history reads a path that git C-quotes (a double quote in the name) in the right file mode', SLOW, () => { + const dir = makeRepo(); + const token = randomString(16, 92); + mkdirSync(path.join(dir, 'we"ird'), { recursive: true }); + write(dir, 'we"ird/a.env', `${['JWT_', 'SECRET'].join('')}=${token}\n`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'quoted path').status).toBe(0); + const result = scan(dir, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain('secret-assignment'); + for (const piece of windows(token)) expect(`${result.stdout}${result.stderr}`).not.toContain(piece); + }); + + it.skipIf(!hasGit())('--history finds a block scalar value added under an unchanged key', SLOW, () => { + const dir = makeRepo(); + const key = ['jwt_', 'secret'].join(''); + const token = randomString(32, 93); + write(dir, 'c.yml', `${key}: >-\n changeme\n`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'a').status).toBe(0); + write(dir, 'c.yml', `${key}: >-\n ${token}\n`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'leak').status).toBe(0); + const result = scan(dir, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain('c.yml secret-assignment x1'); + expect(result.stderr).toContain('in 1 of 2 commits'); + }); + + it.skipIf(!hasGit())('--history exits 0 when the split configuration never held a real value', SLOW, () => { + const dir = makeRepo(); + const name = ['JWT_', 'SECRET'].join(''); + const manifest = (v) => `env:\n - name: ${name}\n value: ${v}\n`; + for (const [v, message] of [['changeme', 'a'], ['your_jwt_secret_here', 'b'], ['changeme', 'c']]) { + write(dir, 'deploy.yaml', manifest(v)); + run('git', ['add', '-A'], dir); + expect(commit(dir, message).status).toBe(0); + } + const result = scan(dir, '--history'); + expect(result.status).toBe(0); + expect(result.stdout).toContain('no hits in 3 commits'); + }); + + + // ---- review round 8: --history/--range context, size-limit wording, staged blob versus an oversize working copy ---- + + // Class: a split name/value pair whose name and value are lines apart (the pair matcher reads YAML/config mappings up to + // PAIR_YAML_LINES lines each way, and JSON/HCL/XML objects up to a few hundred characters). The history reader must keep + // as much unchanged context as the matcher looks across, and blame the commit that added the VALUE line. + const pairName = ['JWT_', 'SECRET'].join(''); + const filler = (count, make) => Array.from({ length: count }, (_, i) => make(i)).join(''); + const SPLIT_LAYOUTS = [ + ['a YAML item, value 10 lines below its name', 'deploy.yaml', (v) => `env:\n - name: ${pairName}\n${filler(9, (i) => ` note${i}: ok\n`)} value: ${v}\n`], + ['a YAML item, value above its name', 'deploy.yaml', (v) => `env:\n - value: ${v}\n${filler(9, (i) => ` note${i}: ok\n`)} name: ${pairName}\n`], + ['a JSON object, value 40 short lines below its name', 'vars.json', (v) => `{\n "name": "${pairName}",\n${filler(40, (i) => ` "n${i}": 1,\n`)} "value": "${v}"\n}\n`], + ['a JSON object, value 40 short lines above its name', 'vars.json', (v) => `{\n "value": "${v}",\n${filler(40, (i) => ` "n${i}": 1,\n`)} "name": "${pairName}"\n}\n`], + ['a mapping key with the value 10 children down', 'secrets.yaml', (v) => `secrets:\n ${pairName}:\n${filler(9, (i) => ` note${i}: ok\n`)} value: ${v}\n`], + ]; + + it.each(SPLIT_LAYOUTS)('--history and --range catch a value added far from an unchanged name (%s)', SLOW, (_label, file, layout) => { + const dir = makeRepo(); + const value = randomString(24, 81); + const save = (v, message) => { + write(dir, file, layout(v)); + run('git', ['add', '-A'], dir); + expect(commit(dir, message).status).toBe(0); + return run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + }; + const first = save('changeme', 'placeholder'); + const leaking = save(value, 'leak'); + const removed = save('changeme', 'remove'); + expect(scan(dir).status).toBe(0); + const firstFull = run('git', ['rev-parse', first], dir).stdout.trim(); + for (const args of [['--history'], ['--range', `${firstFull}..HEAD`]]) { + const result = scan(dir, ...args); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${leaking} ${file} secret-name-value-pair x1`); + expect(result.stderr).not.toContain(removed); + for (const piece of windows(value)) expect(output).not.toContain(piece); + } + }); + + it.skipIf(!hasGit())('--history does not blame a later commit that adds an unrelated line inside the pair window', SLOW, () => { + const dir = makeRepo(); + const value = randomString(24, 82); + const layout = (extra) => `env:\n - name: ${pairName}\n${filler(4, (i) => ` note${i}: ok\n`)}${extra} value: ${value}\n`; + write(dir, 'deploy.yaml', layout('')); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'leak').status).toBe(0); + const leaking = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + write(dir, 'deploy.yaml', layout(' extra: 1\n')); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'unrelated line').status).toBe(0); + const later = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + const result = scan(dir, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain(leaking); + expect(result.stderr).not.toContain(later); + }); + + it.skipIf(!hasGit())('--history still reads a lockfile edited in many places without treating it as oversize', SLOW, () => { + const dir = makeRepo(); + const body = (v) => filler(3000, (i) => ` "dep${i}": { "version": "${v}.${i}" },\n`); + write(dir, 'package-lock.json', `{\n${body(1)}}\n`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'lock').status).toBe(0); + write(dir, 'package-lock.json', `{\n${body(2)}}\n`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'bump').status).toBe(0); + const result = scan(dir, '--history'); + expect(result.status).toBe(0); + expect(result.stdout).toContain('no hits in 2 commits'); + }); + + it('the history context is derived from the pair matcher, not a separate number', () => { + // 12 lines is the YAML window; the character windows (1500 back or forward) can span one line per character (blank lines). + expect(HISTORY_CONTEXT).toBeGreaterThanOrEqual(12); + expect(HISTORY_CONTEXT).toBeGreaterThanOrEqual(1500); + }); + + // Class: the pair matcher looks 1,500 CHARACTERS around a JSON/HCL field, and JSON may have blank or one-character lines, + // so --history/--range must keep unchanged context by characters, not by a count of lines a minimum line length implies. + const GAP_LAYOUTS = [ + ['500 blank lines', () => '\n'.repeat(500)], + ['500 one-character lines', () => ' \n'.repeat(500)], + ['500 CRLF blank lines', () => '\r\n'.repeat(500)], + ['1,400 blank lines', () => '\n'.repeat(1400)], + ['mixed blank, one-character and CRLF lines', () => Array.from({ length: 500 }, (_, i) => ['\n', ' \n', '\r\n', ';\n'][i % 4]).join('')], + ]; + const gapDoc = (gap, v, nameFirst = true) => + nameFirst ? `{"name":"${pairName}",\n${gap}"value":"${v}"}\n` : `{"value":"${v}",\n${gap}"name":"${pairName}"}\n`; + + it.each(GAP_LAYOUTS)('the tree scan pairs a name and a value across %s (the layout the history tests use)', (_label, makeGap) => { + expect(scanText('vars.json', gapDoc(makeGap(), randomString(24, 71))).length).toBeGreaterThan(0); + }); + + it.each(GAP_LAYOUTS)('--history and --range catch a changed value across %s, then a removal', SLOW, (_label, makeGap) => { + if (!hasGit()) return; + for (const nameFirst of [true, false]) { + const dir = makeRepo(); + const value = randomString(24, 72); + const save = (v, message) => { + write(dir, 'vars.json', gapDoc(makeGap(), v, nameFirst)); + run('git', ['add', '-A'], dir); + expect(commit(dir, message).status).toBe(0); + return run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + }; + const first = save('changeme', 'placeholder'); + const leaking = save(value, 'leak'); + save('changeme', 'remove'); + expect(scan(dir).status).toBe(0); + const firstFull = run('git', ['rev-parse', first], dir).stdout.trim(); + for (const args of [['--history'], ['--range', `${firstFull}..HEAD`]]) { + const result = scan(dir, ...args); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status, args[0]).toBe(1); + expect(result.stderr).toContain(`${leaking} vars.json secret-name-value-pair x1`); + for (const piece of windows(value)) expect(output).not.toContain(piece); + } + } + }); + + it.skipIf(!hasGit())('--history and --range catch a value that stays in the tree across blank lines', SLOW, () => { + const dir = makeRepo(); + const value = randomString(24, 73); + write(dir, 'vars.json', gapDoc('\n'.repeat(500), 'changeme')); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'placeholder').status).toBe(0); + const first = run('git', ['rev-parse', 'HEAD'], dir).stdout.trim(); + write(dir, 'vars.json', gapDoc('\n'.repeat(500), value)); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'leak').status).toBe(0); + expect(scan(dir).status).toBe(1); + for (const args of [['--history'], ['--range', `${first}..HEAD`]]) expect(scan(dir, ...args).status, args[0]).toBe(1); + }); + + it.skipIf(!hasGit())('--history does not blame a later commit that adds an unrelated line inside a blank-line gap', SLOW, () => { + const dir = makeRepo(); + const value = randomString(24, 74); + const layout = (extra) => gapDoc(`${'\n'.repeat(250)}${extra}${'\n'.repeat(250)}`, value); + write(dir, 'vars.json', layout('')); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'leak').status).toBe(0); + const leaking = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + write(dir, 'vars.json', layout('\n')); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'one more blank line').status).toBe(0); + const later = run('git', ['rev-parse', '--short=7', 'HEAD'], dir).stdout.trim(); + const result = scan(dir, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain(leaking); + expect(result.stderr).not.toContain(later); + }); + + // Class: the size limit differs by path (5 MB, 16 MB for a lockfile) and the message must name the one that was applied. + it('the oversize report gives each path the limit and the constant that applies to it', () => { + const only = formatOversizeReport(['data/big.json']); + expect(only).toContain('over 5 MB NOT scanned'); + expect(only).toContain('(over 5 MB, MAX_FILE_BYTES)'); + expect(only).not.toContain('16 MB'); + const lock = formatOversizeReport(['package-lock.json']); + expect(lock).toContain('over 16 MB NOT scanned'); + expect(lock).toContain('(over 16 MB, MAX_LOCKFILE_BYTES)'); + expect(lock).toContain('raise MAX_LOCKFILE_BYTES.'); + expect(lock).not.toContain('5 MB'); + const both = formatOversizeReport(['data/big.json', 'sub/yarn.lock']); + expect(both).toContain('data/big.json (over 5 MB, MAX_FILE_BYTES)'); + expect(both).toContain('sub/yarn.lock (over 16 MB, MAX_LOCKFILE_BYTES)'); + expect(both).toContain('raise MAX_FILE_BYTES / MAX_LOCKFILE_BYTES.'); + }); + + it('the history report gives the limit of each kind of oversize version', () => { + const report = formatHistoryReport([], { + commits: 4, + oversize: 3, + unscanned: 3, + oversizeLimits: new Map([[5 * 1024 * 1024, 1], [16 * 1024 * 1024, 2]]), + }); + expect(report).toContain('3 file versions NOT scanned (1 over the 5 MB limit, 2 over the 16 MB lockfile limit)'); + expect(report).toContain('MAX_FILE_BYTES and MAX_LOCKFILE_BYTES'); + const lockOnly = formatHistoryReport([], { commits: 1, oversize: 1, unscanned: 1, oversizeLimits: new Map([[16 * 1024 * 1024, 1]]) }); + expect(lockOnly).toContain('(1 over the 16 MB lockfile limit)'); + expect(lockOnly).not.toContain('5 MB'); + }); + + const LOCK_OVER = 16 * 1024 * 1024 + 4096; + const bigLockfile = () => `{\n${`${'x'.repeat(1023)}\n`.repeat(Math.ceil(LOCK_OVER / 1024))}}\n`; + + it.skipIf(!hasGit())('an oversize lockfile is reported against the lockfile limit, in the tree scan and in --history/--range', SLOW, () => { + const dir = makeRepo(); + write(dir, 'package-lock.json', bigLockfile()); + run('git', ['add', '-A'], dir); + const tree = scan(dir); + expect(tree.status).toBe(1); + expect(tree.stderr).toContain('package-lock.json (over 16 MB, MAX_LOCKFILE_BYTES)'); + expect(tree.stderr).not.toContain('5 MB'); + expect(commit(dir, 'big lockfile').status).toBe(0); + const head = run('git', ['rev-parse', 'HEAD'], dir).stdout.trim(); + for (const args of [['--history'], ['--range', `${'0'.repeat(40)}..${head}`]]) { + const result = scan(dir, ...args); + expect(result.status).toBe(2); + expect(result.stderr).toContain('1 over the 16 MB lockfile limit'); + expect(result.stderr).toContain('MAX_LOCKFILE_BYTES'); + expect(result.stderr).not.toContain('5 MB'); + } + }); + + it.skipIf(!hasGit())('an ordinary oversize file is still reported against MAX_FILE_BYTES in --history', SLOW, () => { + const dir = makeBigVersionRepo(); + const result = scan(dir, '--history'); + expect(result.status).toBe(2); + expect(result.stderr).toContain('1 over the 5 MB limit'); + expect(result.stderr).toContain('MAX_FILE_BYTES'); + expect(result.stderr).not.toContain('MAX_LOCKFILE_BYTES'); + }); + + // Class: the staged (index) blob must be compared with the working tree BEFORE any size/binary shortcut, on every path + // that skips reading the working copy (oversize binary, oversize text, oversize lockfile). + const bigBinary = (extra = 4096) => Buffer.concat([PNG_HEAD, Buffer.alloc(5 * 1024 * 1024 + extra, 1)]); + + it.skipIf(!hasGit())('a staged secret is still found when the working copy becomes a binary over the size limit', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.env', assignment); + run('git', ['add', '-A'], dir); + write(dir, 'x.env', bigBinary()); + const result = scan(dir); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env:1 secret-assignment (index)'); + expect(output).not.toContain('OK ('); + for (const piece of windows(secret)) expect(output).not.toContain(piece); + }); + + it.skipIf(!hasGit())('an unchanged binary over the size limit is still skipped, counted and clean', SLOW, () => { + const dir = makeRepo(); + write(dir, 'big.bin', bigBinary()); + write(dir, 'ok.txt', 'hello\n'); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(0); + expect(result.stdout).toContain('OK (1 files scanned, 1 skipped: 1 binary)'); + }); + + it.skipIf(!hasGit())('a clean staged text file under an oversize binary working copy passes and says the versions differ', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.txt', 'hello\n'); + run('git', ['add', '-A'], dir); + write(dir, 'x.txt', bigBinary()); + const result = scan(dir); + expect(result.status).toBe(0); + expect(result.stdout).toContain('1 with a staged version that differs from the working tree'); + }); + + it.skipIf(!hasGit())('fails closed when the staged blob under an oversize binary working copy is itself oversize', SLOW, () => { + const dir = makeRepo(); + write(dir, 'big.env', overLimitText(assignment)); + run('git', ['add', '-A'], dir); + write(dir, 'big.env', bigBinary()); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('big.env (over 5 MB, MAX_FILE_BYTES)'); + }); + + it.skipIf(!hasGit())('a staged secret is found under an oversize TEXT working copy too, and the oversize file is still reported once', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.env', assignment); + run('git', ['add', '-A'], dir); + write(dir, 'x.env', overLimitText()); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env:1 secret-assignment (index)'); + expect(result.stderr.split('x.env (over 5 MB, MAX_FILE_BYTES)').length - 1).toBe(1); + }); + + it.skipIf(!hasGit())('a staged lockfile credential is found when the working copy becomes a binary over the lockfile limit', SLOW, () => { + const dir = makeRepo(); + const password = randomString(24, 83); + write(dir, 'package-lock.json', `{\n "x": {\n "_authToken": "${password}"\n }\n}\n`); + run('git', ['add', '-A'], dir); + expect(scan(dir).status).toBe(1); // the fixture is a finding on its own + write(dir, 'package-lock.json', Buffer.concat([PNG_HEAD, Buffer.alloc(LOCK_OVER, 1)])); + const result = scan(dir); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status).toBe(1); + expect(result.stderr).toContain('package-lock.json:3 lockfile-credential (index)'); + for (const piece of windows(password)) expect(output).not.toContain(piece); + }); + + // ---- review round 5: nothing tracked may go unexamined ---- + + const RAW_NAME = Buffer.concat([Buffer.from('bad-'), Buffer.from([0xff]), Buffer.from('.env')]); + const rawPath = (dir) => Buffer.concat([Buffer.from(`${dir}${path.sep}`), RAW_NAME]); + const canCreateRawName = (() => { + const probe = mkdtempSync(path.join(tmpdir(), 'check-secrets-probe-')); + try { + writeFileSync(rawPath(probe), 'x'); + return true; + } catch { + return false; + } finally { + rmSync(probe, { recursive: true, force: true }); + } + })(); + const RAW_SKIP_REASON = 'this file system cannot create a file name that is not valid UTF-8'; + + it.skipIf(!hasGit() || !canCreateRawName)(`scans a tracked file whose name is not valid UTF-8 (${RAW_SKIP_REASON})`, SLOW, () => { + const dir = makeRepo(); + writeFileSync(rawPath(dir), assignment); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain(':1 secret-assignment'); + expect(result.stderr).toContain('bad-\\xff.env'); + expect(result.stdout).not.toContain('OK'); + }); + + it.skipIf(!hasGit() || !canCreateRawName)(`a clean non-UTF-8 file name is counted as scanned, not skipped (${RAW_SKIP_REASON})`, SLOW, () => { + const dir = makeRepo(); + writeFileSync(rawPath(dir), 'A=1\n'); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(0); + expect(result.stdout).toMatch(/1 files scanned, 0 skipped/); + }); + + it.skipIf(!hasGit())('fails, and names the count, when a tracked file exists but cannot be read (here: a directory took its place)', SLOW, () => { + const dir = makeRepo(); + write(dir, 'config.env', assignment); + write(dir, 'ok.js', 'export const x = 1;\n'); + run('git', ['add', '-A'], dir); + rmSync(path.join(dir, 'config.env')); + mkdirSync(path.join(dir, 'config.env')); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('1 tracked file could NOT be read'); + expect(result.stderr).toContain('config.env'); + expect(result.stdout).not.toContain('OK'); + }); + + it.skipIf(!hasGit())('a tracked file that is deleted in the working tree is scanned from the index, so a staged secret still fails', SLOW, () => { + const dir = makeRepo(); + write(dir, 'gone.env', assignment); + run('git', ['add', '-A'], dir); + rmSync(path.join(dir, 'gone.env')); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('gone.env:1 secret-assignment'); + }); + + it.skipIf(!hasGit())('a clean deleted-but-listed file passes and the summary says it came from the index', SLOW, () => { + const dir = makeRepo(); + write(dir, 'gone.env', 'A=1\n'); + write(dir, 'kept.js', 'export const x = 1;\n'); + run('git', ['add', '-A'], dir); + rmSync(path.join(dir, 'gone.env')); + const result = scan(dir); + expect(result.status).toBe(0); + expect(result.stdout).toContain('2 files scanned'); + expect(result.stdout).toContain('1 missing from the working tree and scanned from the index'); + }); + + it.skipIf(!hasGit() || process.platform === 'win32')('scans the target text of a tracked symlink instead of skipping it', SLOW, () => { + const dir = makeRepo(); + symlinkSync(assignment.trim(), path.join(dir, 'link.env')); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('link.env:1 secret-assignment'); + }); + + it('the unreadable report names the count and paths only', () => { + const report = formatUnreadableReport(['a.env', 'b.env']); + expect(report).toContain('2 tracked files could NOT be read'); + expect(report).toContain(' b.env'); + }); + + function makeBigVersionRepo() { + const dir = makeRepo(); + write(dir, 'big.env', overLimitText(assignment)); // just over 5 MB, secret on line 1 + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add big file').status).toBe(0); + write(dir, 'big.env', 'X=1\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'shrink it').status).toBe(0); + return dir; + } + + it.skipIf(!hasGit())('--history is NOT clean (exit 2, no "no hits") when an added version was too large to scan', SLOW, () => { + const dir = makeBigVersionRepo(); + const result = scan(dir, '--history'); + expect(result.status).toBe(2); + expect(`${result.stdout}${result.stderr}`).not.toContain('no hits'); + expect(result.stderr).toContain('INCOMPLETE'); + expect(result.stderr).toContain('1 file version NOT scanned'); + expect(result.stderr).toContain('1 over the 5 MB limit'); + }); + + it.skipIf(!hasGit())('--history still reports hits (exit 1) and the gap when both happen', SLOW, () => { + const dir = makeBigVersionRepo(); + write(dir, '.env', assignment); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add config').status).toBe(0); + const result = scan(dir, '--history'); + expect(result.status).toBe(1); + expect(result.stderr).toContain('.env secret-assignment x1'); + expect(result.stderr).toContain('NOT scanned'); + }); + + it.skipIf(!hasGit())('--history on a shallow clone is incomplete (exit 2), not "no hits"', SLOW, () => { + const source = makeRepo(); + write(source, 'a.txt', 'one\n'); + run('git', ['add', '-A'], source); + expect(commit(source, 'one').status).toBe(0); + write(source, 'a.txt', 'two\n'); + run('git', ['add', '-A'], source); + expect(commit(source, 'two').status).toBe(0); + const shallow = path.join(source, '..', `${path.basename(source)}-shallow`); + dirs.push(shallow); + expect(run('git', ['clone', '-q', '--depth=1', `file://${source}`, shallow], source).status).toBe(0); + const result = scan(shallow, '--history'); + expect(result.status).toBe(2); + expect(result.stderr).toContain('shallow clone'); + expect(`${result.stdout}${result.stderr}`).not.toContain('no hits'); + }); + + it('the history report counts versions that were not scanned', () => { + const report = formatHistoryReport([{ commit: 'a'.repeat(40), path: '.env', rule: 'jwt-token', count: 1 }], { + commits: 3, + oversize: 1, + unscanned: 2, + }); + expect(report).toContain('2 file versions NOT scanned (1 over the 5 MB limit)'); + }); + + const utf32 = (text, { bom = false, littleEndian = true } = {}) => { + const cps = [...text].map((c) => c.codePointAt(0)); + const out = Buffer.alloc(cps.length * 4); + cps.forEach((cp, i) => (littleEndian ? out.writeUInt32LE(cp, i * 4) : out.writeUInt32BE(cp, i * 4))); + const mark = littleEndian ? [0xff, 0xfe, 0, 0] : [0, 0, 0xfe, 0xff]; + return bom ? Buffer.concat([Buffer.from(mark), out]) : out; + }; + + it.skipIf(!hasGit())('a stray NUL byte does not exempt a text file (before or after the secret)', SLOW, () => { + for (const content of [`\0${assignment}`, `${assignment}\0`, `FOO=1\n\0\n${assignment}`]) { + const dir = makeRepo(); + write(dir, '.env', content); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('.env:'); + expect(result.stdout).not.toContain('OK'); + } + }); + + it.skipIf(!hasGit())('scans UTF-32 files with and without a BOM', SLOW, () => { + for (const options of [{ bom: true }, { bom: true, littleEndian: false }, {}, { littleEndian: false }]) { + const dir = makeRepo(); + write(dir, '.env', utf32(assignment, options)); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status, JSON.stringify(options)).toBe(1); + expect(result.stderr).toContain('.env:1 secret-assignment'); + } + }); + + it.skipIf(!hasGit())('an over-limit file with a NUL near the start is reported as oversize, not skipped as binary', SLOW, () => { + const dir = makeRepo(); + write(dir, 'big.env', overLimitText(`\0${assignment}`)); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('big.env'); + expect(result.stderr).toContain('NOT scanned'); + }); + + // ------------------------------------------------------------------------- + // Review round 7 (finding 6): the staged (index) version and the working-tree version are both scanned. + // + // Class: the two versions of a tracked path can differ. Siblings: secret staged then swapped for a placeholder in + // the working tree; secret only in the working tree; different secrets in each; several files where one differs; + // an unmerged path (three stages) resolved to a placeholder in the working tree; an index blob over the size limit + // under a small working-tree file; line-ending conversion (both versions clean); a deleted working-tree file; a + // SHA-256 repository; identical versions (a CI checkout) cost nothing extra and print nothing extra. + // ------------------------------------------------------------------------- + + const placeholderLine = `${['API_', 'KEY'].join('')}=your_api_key_here\n`; + + it.skipIf(!hasGit())('scans a secret that is staged even when the working-tree file was swapped for a placeholder', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.env', assignment); + run('git', ['add', '-A'], dir); + write(dir, 'x.env', placeholderLine); + const result = scan(dir); + const output = `${result.stdout}\n${result.stderr}`; + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env:1 secret-assignment (index)'); + expect(result.stderr).not.toContain('(working tree)'); + for (const piece of windows(secret)) expect(output).not.toContain(piece); + }); + + it.skipIf(!hasGit())('scans a secret that exists only in the working tree, and says which version it was in', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.env', placeholderLine); + run('git', ['add', '-A'], dir); + write(dir, 'x.env', assignment); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env:1 secret-assignment (working tree)'); + expect(result.stderr).not.toContain('(index)'); + }); + + it.skipIf(!hasGit())('reports both versions when both hold a (different) secret', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.env', assignment); + run('git', ['add', '-A'], dir); + write(dir, 'x.env', `${['API_', 'KEY'].join('')}=${randomString(32, 262)}\n`); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env:1 secret-assignment (index)'); + expect(result.stderr).toContain('x.env:1 secret-assignment (working tree)'); + }); + + it.skipIf(!hasGit())('an ordinary checkout (index and working tree identical) prints no version labels and no extra note', SLOW, () => { + const dir = makeRepo(); + write(dir, 'a.env', placeholderLine); + write(dir, 'src/app.js', 'export const x = 1;\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'add').status).toBe(0); + const clean = scan(dir); + expect(clean.status).toBe(0); + expect(clean.stdout).not.toContain('differs'); + write(dir, 'b.env', assignment); + run('git', ['add', '-A'], dir); + const dirty = scan(dir); + expect(dirty.status).toBe(1); + expect(dirty.stderr).toContain('b.env:1 secret-assignment'); + expect(dirty.stderr).not.toContain('(index)'); + expect(dirty.stderr).not.toContain('(working tree)'); + }); + + it.skipIf(!hasGit())('finds the one staged secret among many files whose working-tree copies differ', SLOW, () => { + const dir = makeRepo(); + for (let i = 0; i < 40; i += 1) write(dir, `cfg/f${i}.env`, i === 23 ? assignment : `A=${i}\n`); + run('git', ['add', '-A'], dir); + for (let i = 0; i < 40; i += 1) write(dir, `cfg/f${i}.env`, placeholderLine); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('1 potential secret found'); + expect(result.stderr).toContain('cfg/f23.env:1 secret-assignment (index)'); + }); + + it.skipIf(!hasGit())('says how many files differ when both versions are clean, and still exits 0', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.env', 'A=1\n'); + run('git', ['add', '-A'], dir); + write(dir, 'x.env', 'A=2\n'); + const result = scan(dir); + expect(result.status).toBe(0); + expect(result.stdout).toContain('1 with a staged version that differs from the working tree (both scanned)'); + }); + + it.skipIf(!hasGit())('line-ending conversion alone (core.autocrlf) makes a file differ, and both versions scan clean', SLOW, () => { + const dir = makeRepo(); + run('git', ['config', 'core.autocrlf', 'true'], dir); + write(dir, 'x.env', 'A=1\r\nB=2\r\n'); + run('git', ['add', '-A'], dir); + const result = scan(dir); + expect(result.status).toBe(0); + }); + + it.skipIf(!hasGit())('a secret staged in a file that was then deleted from the working tree is still found (index only)', SLOW, () => { + const dir = makeRepo(); + write(dir, 'gone.env', assignment); + run('git', ['add', '-A'], dir); + rmSync(path.join(dir, 'gone.env')); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('gone.env:1 secret-assignment (index)'); + }); + + it.skipIf(!hasGit())('an unmerged path is scanned in every stage, even when the working tree was resolved to a placeholder', SLOW, () => { + const dir = makeRepo(); + write(dir, 'x.env', 'A=1\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'base').status).toBe(0); + run('git', ['checkout', '-q', '-b', 'other'], dir); + write(dir, 'x.env', `A=3\n${assignment}`); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'other side').status).toBe(0); + run('git', ['checkout', '-q', '-'], dir); + write(dir, 'x.env', 'A=2\n'); + run('git', ['add', '-A'], dir); + expect(commit(dir, 'this side').status).toBe(0); + expect(git(dir, 'merge', 'other').status).toBe(1); // conflict (git exits 1): stages 1, 2 and 3 are in the index + write(dir, 'x.env', placeholderLine); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env:2 secret-assignment (index)'); + }); + + it.skipIf(!hasGit())('an index blob over the size limit under a small working-tree file is reported, not skipped', SLOW, () => { + const dir = makeRepo(); + write(dir, 'big.env', overLimitText(assignment)); + run('git', ['add', '-A'], dir); + write(dir, 'big.env', 'A=1\n'); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('NOT scanned'); + expect(result.stderr).toContain('big.env'); + }); + + it.skipIf(!hasGit())('works in a SHA-256 repository (the blob id is computed with the repository hash)', SLOW, () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-')); + dirs.push(dir); + if (run('git', ['init', '-q', '--object-format=sha256'], dir).status !== 0) return; // git older than 2.29 + write(dir, 'x.env', assignment); + run('git', ['add', '-A'], dir); + const staged = scan(dir); + expect(staged.status).toBe(1); + expect(staged.stderr).not.toContain('(index)'); // identical versions: no label + write(dir, 'x.env', placeholderLine); + const swapped = scan(dir); + expect(swapped.status).toBe(1); + expect(swapped.stderr).toContain('x.env:1 secret-assignment (index)'); + }); + + it('the report labels the version and still holds only path, line and rule', () => { + const report = formatReport([ + { path: 'a.env', line: 3, rule: 'secret-assignment', source: 'index' }, + { path: 'a.env', line: 3, rule: 'secret-assignment', source: 'working tree' }, + { path: 'b.env', line: 1, rule: 'jwt-token' }, + ]); + expect(report).toContain(' a.env:3 secret-assignment (index)'); + expect(report).toContain(' a.env:3 secret-assignment (working tree)'); + expect(report).toContain(' b.env:1 jwt-token\n'); + }); + + it('--help exits 0 and unknown arguments exit 2', SLOW, () => { + const help = scan(REPO_ROOT, '--help'); + expect(help.status).toBe(0); + expect(help.stdout).toContain('Usage'); + const bad = scan(REPO_ROOT, '--values'); + expect(bad.status).toBe(2); + expect(bad.stderr).toContain('unknown argument'); + }); +}); + + +// --------------------------------------------------------------------------- +// Review round 9: lockfiles, commit ranges in CI, XML configuration, provider token families +// +// (1) Lockfiles were skipped by name, so a credential in a dependency URL or an auth field was never seen. They are now +// scanned with targeted rules (URL credentials, auth fields, provider tokens, webhooks, private keys) after ordinary +// integrity digests are blanked. Siblings: package-lock.json, npm-shrinkwrap.json, yarn.lock (v1 and berry), +// pnpm-lock.yaml, Cargo.lock, Gemfile.lock, poetry.lock, composer.lock, go.sum, Pipfile.lock, bun.lock; the fields +// resolved / tarball / url / source / remote / registry; URL password, token as user name, credential query parameter, +// _authToken / _auth / _password fields; and --history / --range, which read lockfile versions the same way. +// (2) --range .., which CI runs so a secret committed and removed inside a pull request is still found. +// (3) XML configuration files were scanned in code mode (a passphrase with spaces was not a finding). .xml, .config, +// MSBuild, .plist, .resx, .wsdl and the Maven, Ant, Tomcat and Android files get configuration-value semantics; SVG +// and HTML stay markup. +// (4) The provider token list: the whole Slack family and the other current families (see PROVIDER_TOKENS). +// --------------------------------------------------------------------------- + +const PREFIX_FLOOD_LIMIT_MS = 30_000; +const ALL_HOSTILE_PREFIXES = ['xapp-', 'xoxe.xoxe.', 'xoxo-', 'xoxe-1-', 'xoxb-', 'glpat-', 'sk-', 'sk-ant-', 'sk-proj-', 'hf_', 'secret_', 'ntn_', 'whsec_', 'AccountKey=', 'sig=', 'https://a', 'dp.st.', 'hvs.', 'ATATT3', 'ATBB', 'SG.', 'ya29.', 'GOCSPX-', 'key-', 'lin_api_', 'HEROKU_API_KEY=', 'DD_API_KEY:', 'a.atlasv1.', 'cfut_', 'sq0csp-', 'sk.eyJ', 'NRII-', 'APA91b', 'AAAA1234567:APA91b', '123456789:AA', 'M', 'shpat_', 'npm_', 'github_pat_', 'pypi-AgEIcHlwaS5vcmc', 'sntrys_']; + +describe('review round 9', () => { + const rulesOf = (file, text) => scanText(file, text).map((f) => f.rule); + const NAME = ['JWT_', 'SECRET'].join(''); + const PASSWORD = ['pass', 'word'].join(''); + const passphrase = 'correct horse battery staple'; + const random = randomString(24, 2001); + const scheme = ['https', '://'].join(''); + const sha512 = (seed) => `sha512-${randomString(86, seed, `${ALNUM}+/`)}==`; + const sha1 = (seed) => `sha1-${randomString(27, seed, `${ALNUM}+/`)}=`; + const hex = (n, seed) => randomString(n, seed, HEX); + + describe('(1) lockfiles', () => { + // Ordinary content of every lockfile format, integrity hashes included. None of it may be reported. + const ORDINARY = [ + ['package-lock.json', JSON.stringify({ name: 'x', lockfileVersion: 3, packages: { '': { dependencies: { password: '^1.0.0', token: 'latest', secret: 'npm:other@1' } }, 'node_modules/pkg': { version: '1.0.0', resolved: `${scheme}registry.npmjs.org/pkg/-/pkg-1.0.0.tgz`, integrity: sha512(2101) }, 'node_modules/git-dep': { version: '1.0.0', resolved: `git+ssh://git@github.com/org/repo.git#${hex(40, 2102)}` } } }, null, 2)], + ['npm-shrinkwrap.json', JSON.stringify({ dependencies: { a: { version: '1.0.0', resolved: `${scheme}registry.npmjs.org/a/-/a-1.0.0.tgz?cache=1`, integrity: sha512(2103) } } }, null, 2)], + ['yarn.lock', `"a@^1.0.0":\n version "1.0.0"\n resolved "${scheme}registry.yarnpkg.com/a/-/a-1.0.0.tgz#${hex(40, 2104)}"\n integrity ${sha512(2105)}\n\n"b@^2":\n version "2.0.0"\n resolved "${scheme}registry.yarnpkg.com/b/-/b-2.0.0.tgz#${hex(40, 2106)}"\n integrity ${sha1(2107)}\n`], + ['yarn.lock', `__metadata:\n version: 8\n"a@npm:^1.0.0":\n version: 1.0.0\n resolution: "a@npm:1.0.0"\n checksum: 10c0/${hex(128, 2108)}\n languageName: node\n linkType: hard\n`], + ['pnpm-lock.yaml', `lockfileVersion: '9.0'\npackages:\n a@1.0.0:\n resolution: {integrity: ${sha512(2109)}, tarball: ${scheme}registry.npmjs.org/a/-/a-1.0.0.tgz}\n`], + ['Cargo.lock', `[[package]]\nname = "a"\nversion = "1.0.0"\nsource = "registry+${scheme}github.com/rust-lang/crates.io-index"\nchecksum = "${hex(64, 2110)}"\n\n[[package]]\nname = "b"\nversion = "0.1.0"\nsource = "git+${scheme}github.com/org/b#${hex(40, 2111)}"\n`], + ['Gemfile.lock', `GEM\n remote: ${scheme}rubygems.org/\n specs:\n rake (13.0.6)\n\nCHECKSUMS\n rake (13.0.6) sha256=${hex(64, 2112)}\n`], + ['poetry.lock', `[[package]]\nname = "requests"\nversion = "2.31.0"\n[package.source]\ntype = "legacy"\nurl = "${scheme}pypi.org/simple"\n[[package.files]]\nfile = "requests-2.31.0.tar.gz"\nhash = "sha256:${hex(64, 2113)}"\n`], + ['composer.lock', JSON.stringify({ packages: [{ name: 'a/b', version: '1.0.0', source: { type: 'git', url: `${scheme}github.com/a/b.git`, reference: hex(40, 2114) }, dist: { type: 'zip', url: `${scheme}api.github.com/repos/a/b/zipball/${hex(40, 2115)}`, shasum: '' } }] }, null, 2)], + ['go.sum', `example.com/a v1.0.0 h1:${randomString(43, 2116, `${ALNUM}+/`)}=\nexample.com/a v1.0.0/go.mod h1:${randomString(43, 2117, `${ALNUM}+/`)}=\n`], + ['Pipfile.lock', JSON.stringify({ default: { requests: { hashes: [`sha256:${hex(64, 2118)}`], index: 'pypi', version: '==2.31.0' } } }, null, 2)], + ['bun.lock', `{\n "lockfileVersion": 1,\n "packages": {\n "a": ["a@1.0.0", "", {}, "${sha512(2119)}"],\n }\n}\n`], + ]; + it.each(ORDINARY)('an ordinary %s (integrity hashes, git dependency, registry URLs) is clean', (file, text) => { + expect(scanText(file, text)).toEqual([]); + }); + + it('the four real lockfiles of this repository are clean', () => { + for (const file of ['package-lock.json', 'app/package-lock.json', 'server/package-lock.json', 'shared/package-lock.json']) { + expect(scanText(file, readFileSync(path.join(REPO_ROOT, file), 'utf8')), file).toEqual([]); + } + }); + + // The credential shapes, in the form each lockfile format writes its resolved URL. + const URL_LOCATIONS = [ + ['package-lock.json', (u) => `{"packages":{"node_modules/x":{"resolved":"${u}","integrity":"${sha512(2120)}"}}}\n`], + ['npm-shrinkwrap.json', (u) => `{"dependencies":{"x":{"version":"1.0.0","resolved":"${u}"}}}\n`], + ['yarn.lock', (u) => `"x@^1":\n version "1.0.0"\n resolved "${u}#${hex(40, 2121)}"\n`], + ['pnpm-lock.yaml', (u) => `packages:\n x@1.0.0:\n resolution: {tarball: ${u}}\n`], + ['Cargo.lock', (u) => `[[package]]\nname = "x"\nsource = "registry+${u}"\n`], + ['Gemfile.lock', (u) => `GEM\n remote: ${u}\n specs:\n x (1.0.0)\n`], + ['poetry.lock', (u) => `[package.source]\ntype = "legacy"\nurl = "${u}"\n`], + ['composer.lock', (u) => `{"packages":[{"dist":{"type":"zip","url":"${u}"}}]}\n`], + ['Pipfile.lock', (u) => `{"_meta":{"sources":[{"name":"internal","url":"${u}"}]}}\n`], + ['bun.lock', (u) => `{"packages":{"x":["x@${u}","",{}]}}\n`], + ]; + const SHAPES = [ + ['password', `${scheme}user:${random}@registry.internal/x/-/x-1.0.0.tgz`, ['url-password', 'lockfile-credential']], + ['hex password', `${scheme}user:${hex(64, 2122)}@registry.internal/x/-/x-1.0.0.tgz`, ['url-password']], + ['token as the user name', `${scheme}${random}@registry.internal/x/-/x-1.0.0.tgz`, ['lockfile-credential']], + ['token query parameter', `${scheme}registry.internal/x/-/x-1.0.0.tgz?token=${random}`, ['lockfile-credential']], + ['_authToken query parameter', `${scheme}registry.internal/x/-/x-1.0.0.tgz?_authToken=${random}`, ['lockfile-credential']], + ['signed URL', `${scheme}registry.internal/x/-/x-1.0.0.tgz?sig=${random}`, ['lockfile-credential']], + ]; + describe.each(URL_LOCATIONS)('%s', (file, place) => { + it.each(SHAPES)('finds a credential in a dependency URL: %s', (_label, url, expected) => { + const found = rulesOf(file, place(url)); + expect(found.some((rule) => expected.includes(rule)), `${file}: ${found.join(',')}`).toBe(true); + }); + }); + + it.each([ + ['a JSON _authToken field', 'package-lock.json', `{"registries":{"//registry.internal/:_authToken":"${random}"},"_authToken":"${random}"}\n`], + ['an ini _authToken line', 'yarn.lock', `//registry.internal/:_authToken=${random}\n`], + ['an _auth line', 'yarn.lock', `_auth = ${Buffer.from(`u:${random}`).toString('base64')}\n`], + ['a _password field', 'package-lock.json', `{"_password":"${random}"}\n`], + ['a yarn berry npmAuthToken', 'yarn.lock', `npmAuthToken: ${random}\n`], + ['a password field in TOML', 'poetry.lock', `password = "${random}"\n`], + ['a bare token field in YAML', 'pnpm-lock.yaml', `token: ${random}\n`], + ])('finds %s', (_label, file, text) => { + expect(rulesOf(file, text)).toContain('lockfile-credential'); + }); + + it('a placeholder, an environment reference or a dependency specifier in an auth-like field is not a finding', () => { + for (const value of ['${NPM_TOKEN}', '', 'your_token_here', '^1.2.3', 'npm:other@1', 'link:../password', 'workspace:*', 'latest', '']) { + expect(rulesOf('package-lock.json', `{"_authToken":"${value}","password":"${value}"}\n`), value).toEqual([]); + } + }); + + it('a provider token, a private key block and a webhook URL in a lockfile are found', () => { + const token = PROVIDER_TOKENS.find(([rule]) => rule === 'github-token' || rule === 'gitlab-token')[2](); + expect(rulesOf('yarn.lock', `# ${token}\n`)).toContain('gitlab-token'); + const pem = ['-----BEGIN ', 'PRIVATE KEY-----'].join(''); + expect(rulesOf('package-lock.json', `{"note":"${pem}\\n${randomString(60, 2123, BASE64)}"}\n`)).toContain('private-key-block'); + const hook = `${scheme}hooks.slack.com/services/T0123ABCD/B0123ABCD/${randomString(24, 2124)}`; + expect(rulesOf('Cargo.lock', `source = "${hook}"\n`)).toContain('webhook-url'); + }); + + it('the generic rules do not run on a lockfile: a hash-like value under a secret-like name is not reported', () => { + expect(rulesOf('package-lock.json', `{"apiKeyHash":"${hex(64, 2125)}","secretToken":"${random}"}\n`)).toEqual([]); + }); + + it('blanking a digest keeps the text length and the line structure, and leaves a hex password in a URL visible', () => { + const text = `a ${sha512(2126)} b\nchecksum = "${hex(64, 2127)}"\nx: ${scheme}u:${hex(64, 2128)}@h/y\n`; + const cleaned = sanitizeLockfile(text); + expect(cleaned.length).toBe(text.length); + expect(cleaned.split('\n').length).toBe(text.split('\n').length); + expect(cleaned).not.toContain('sha512-'); + expect(cleaned.split('\n')[2]).toBe(text.split('\n')[2]); + expect(cleaned.split('\n')[1]).toBe(`checksum = "${' '.repeat(64)}"`); + }); + + it('the allow marker works on a lockfile line', () => { + const line = `{"resolved":"${scheme}${random}@registry.internal/x.tgz"} // ${ALLOW_MARKER}\n`; + expect(scanText('package-lock.json', line)).toEqual([]); + }); + + it('the hostile-input limit holds on lockfile-shaped input (2 MB each)', SLOW, () => { + const timings = timeInChild(` + const repeat = (unit) => unit.repeat(Math.ceil((2 * 1024 * 1024) / unit.length)); + for (const text of [repeat('https://a:'), repeat('sha512-'), repeat('_authToken='), repeat('checksum = "a'), repeat('password: '), repeat('git+ssh://a@'), repeat('a://b@c?token='), repeat('password: 1 '), repeat('token=' + ' '.repeat(50))]) { + const started = performance.now(); + scanText('package-lock.json', text); + timings.push(Math.round(performance.now() - started)); + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + describe('(3) XML configuration files', () => { + const XML_FILES = [ + 'settings.xml', 'pom.xml', 'conf/server.xml', 'web.config', 'app.config', 'App.csproj', 'Directory.Build.props', 'x.targets', + 'Info.plist', 'Strings.resx', 'service.wsdl', 'app/src/main/res/values/strings.xml', 'build.xml', + 'App.vbproj', 'App.fsproj', 'pkg.nuspec', 'Profile.pubxml', 'Settings.settings', + ]; + it.each(XML_FILES)('%s is classified as configuration', (file) => { + expect(fileMode(file)).toBe('config'); + expect(isXmlConfigPath(file)).toBe(true); + }); + it.each(['logo.svg', 'index.html', 'page.htm', 'doc.xhtml', 'notes.md', 'app.js'])('%s stays out of the XML class', (file) => { + expect(isXmlConfigPath(file)).toBe(false); + expect(fileMode(file)).not.toBe('config'); + }); + + const SHAPES = [ + ['property/name/value', (v) => `${NAME}${v}`], + ['multi-line property', (v) => `\n ${NAME}\n ${v}\n`], + ['add key/value', (v) => ``], + ['attribute', (v) => ``], + ['Maven server password', (v) => `repobob<${PASSWORD}>${v}`], + ['Maven proxy passphrase element', (v) => `<${['pass', 'phrase'].join('')}>${v}`], + ['Ant property', (v) => ``], + ['Tomcat connector', (v) => ``], + ['plist key/string', (v) => `API_SECRET${v}`], + ['Android string', (v) => `${v}`], + ['resx data', (v) => `${v}`], + ]; + describe.each(XML_FILES)('in %s', (file) => { + it.each(SHAPES)('a passphrase with spaces is found: %s', (_label, shape) => { + expect(scanText(file, `${shape(passphrase)}\n`).length).toBeGreaterThan(0); + }); + it.each(SHAPES)('a random value is found: %s', (_label, shape) => { + expect(scanText(file, `${shape(random)}\n`).length).toBeGreaterThan(0); + }); + }); + + it('the reported case: JWT_SECRET in settings.xml is found exactly like in a .config file', () => { + const text = `${NAME}${passphrase}\n`; + expect(rulesOf('settings.xml', text)).toEqual(rulesOf('a.config', text)); + expect(rulesOf('settings.xml', text)).toContain('secret-name-value-pair'); + }); + + it('svg and html keep code-mode semantics: markup is not noisy', () => { + const markup = `\n<${PASSWORD}>${passphrase}`; + expect(scanText('form.html', markup)).toEqual([]); + expect(scanText('icon.svg', `the token is shown here<${PASSWORD}>${passphrase}`)).toEqual([]); + }); + + it.each([ + ['a pom.xml without credentials', 'pom.xml', 'org.xy1.0.0Reads the token from the environment'], + ['Maven environment references', 'settings.xml', `x<${PASSWORD}>\${env.REPO_PASSWORD}`], + ['a Maven-encrypted password', 'settings.xml', `<${PASSWORD}>{${randomString(44, 2201, `${ALNUM}+/=`)}}`], + ['a placeholder', 'settings.xml', `<${PASSWORD}>changeme<${PASSWORD}>your_password_here<${PASSWORD}>`], + ['an Android layout', 'res/layout/login.xml', ``], + ['Android UI strings', 'app/src/main/res/values/strings.xml', `Reset your ${PASSWORD}Enter your ${PASSWORD}Passwords do not match`], + ['a sitemap', 'sitemap.xml', 'https://example.com/a2024-01-01'], + ['an SVG-like .xml image', 'icon.xml', ''], + ['a web.config with connection string references', 'web.config', ``], + ['a plist with ordinary keys', 'Info.plist', 'CFBundleNameLocal CatchNSCameraUsageDescriptionScan a label with the camera'], + ])('%s is clean', (_label, file, text) => { + expect(scanText(file, `${text}\n`)).toEqual([]); + }); + + it('scans XML in linear time (element matcher on 1 MB)', SLOW, () => { + const timings = timeInChild(` + const P = ${JSON.stringify(PASSWORD)}; + const cases = [' '.repeat(1024 * 1024), '

'.repeat(300_000), ('<' + P + '>').repeat(150_000), ('<' + P + ' ' + 'a=1 '.repeat(75) + '>').repeat(2000), ('').repeat(100_000), ('' + '<' + 'x:'.repeat(30) + 'a>').repeat(20_000)]; + for (const text of cases) { + for (const file of ['settings.xml', 'a.vcxproj', 'a.xml.template']) { + const started = performance.now(); + scanText(file, text); + timings.push(Math.round(performance.now() - started)); + } + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + describe('(4) provider token families', () => { + it.each(PROVIDER_TOKENS)('%s: %s is found in JSON, Markdown, source, env and YAML under a non-secret name', (rule, _label, build) => { + const token = build(); + for (const [file, text] of [ + ['data.json', `{"note":"${token}"}\n`], + ['notes.md', `Use ${token} for the job.\n`], + ['src/app.js', `const banner = "${token}";\n`], + ['app.env', `SOMETHING_ELSE=${token}\n`], + ['ci.yaml', `description: ${token}\n`], + ]) { + expect(rulesOf(file, text), `${file} ${_label}`).toContain(rule); + } + }); + + it('nothing but path, line and rule is reported for a provider token', () => { + for (const [, , build, secretPart] of PROVIDER_TOKENS) { + const token = build(); + const report = formatReport(scanText('notes.md', `x ${token}\n`)); + for (const piece of windows(secretPart ? secretPart(token) : token.slice(4), 10).slice(0, 6)) expect(report).not.toContain(piece); + } + }); + + it('the rest of the GitHub and Stripe families is found too', () => { + const r = (n, seed) => randomString(n, seed); + for (const token of [ + ['gh', 'u_', r(36, 2301)].join(''), + ['gh', 's_', r(36, 2302)].join(''), + ['gh', 'r_', r(36, 2303)].join(''), + ['gh', 'o_', r(36, 2304)].join(''), + ['github', '_pat_', r(22, 2305), '_', r(59, 2306)].join(''), + ['r', 'k_live_', r(24, 2307)].join(''), + ['s', 'k_live_', r(24, 2308)].join(''), + ]) { + expect(rulesOf('notes.md', `x ${token}\n`).length, 'a provider token').toBeGreaterThan(0); + } + }); + + it('the reported case: an app-level Slack token in JSON, Markdown and source', () => { + const token = PROVIDER_TOKENS.find(([, label]) => label.startsWith('Slack app-level'))[2](); + expect(rulesOf('a.json', `{"note":"${token}"}\n`)).toContain('slack-token'); + expect(rulesOf('a.md', `token ${token}\n`)).toContain('slack-token'); + expect(rulesOf('a.js', `const t = "${token}";\n`)).toContain('slack-token'); + }); + + const pad = (prefix, n, ch = 'x') => `${prefix}${ch.repeat(n)}`; + const PLACEHOLDERS = [ + pad(['xapp', '-1-A0000000000-0000000000000-'].join(''), 40), + pad(['xox', 'b-000000000000-'].join(''), 24), + pad(['xoxe', '.xoxp-1-'].join(''), 40), + pad(['gl', 'pat-'].join(''), 24), + pad(['npm', '_'].join(''), 36), + pad(['pypi', '-AgEIcHlwaS5vcmc'].join(''), 60), + pad(['whsec', '_'].join(''), 32), + pad(['S', 'K'].join(''), 32, '0'), + pad(['key', '-'].join(''), 32, '0'), + pad(['shp', 'at_'].join(''), 32, '0'), + pad(['do', 'p_v1_'].join(''), 64, '0'), + pad(['hf', '_'].join(''), 34), + pad(['sk', '-proj-'].join(''), 48), + pad(['sk', '-ant-api03-'].join(''), 60), + pad(['sk', '-'].join(''), 48), + pad(['ya', '29.'].join(''), 40), + pad(['GOC', 'SPX-'].join(''), 28), + pad(['HR', 'KU-'].join(''), 40), + pad(['sntr', 'ys_'].join(''), 60), + pad(['dp', '.st.dev.'].join(''), 44), + pad(['hv', 's.'].join(''), 40), + pad(['lin', '_api_'].join(''), 40), + pad(['nt', 'n_'].join(''), 46), + pad(['AT', 'ATT3'].join(''), 60), + ['xapp', '-your-slack-app-token-goes-here'].join(''), + ['glpat', '-'].join(''), + ['sk', '-ant-...'].join(''), + `${['xapp', '-1-'].join('')}\${SLACK_APP_TOKEN}`, + ]; + it.each(PLACEHOLDERS.map((v) => [v.slice(0, 14), v]))('placeholder %s... passes', (_label, value) => { + for (const [file, text] of [['a.json', `{"note":"${value}"}\n`], ['a.md', `token ${value}\n`], ['a.js', `const t = "${value}";\n`], ['.env', `X=${value}\n`]]) { + expect(rulesOf(file, text).filter((rule) => PROVIDER_TOKENS.some(([known]) => known === rule)), `${file}: ${_label}`).toEqual([]); + } + }); + + it.each([ + 'Slack app-level tokens start with xapp- and bot tokens with xoxb-, see the Slack docs.', + 'A GitLab token starts with glpat- and a Hugging Face token with hf_.', + 'Set ntn_ or secret_ as the prefix; Vault uses hvs. for service tokens and Doppler dp.st. for service tokens.', + 'Use the sk-ant- prefix check and the whsec_ prefix check in the validator.', + 'xoxo-love-and-hugs-from-the-team-to-you', + 'the key-value store and the key-value-config-option-name-that-is-long', + 'const cache = process.env.npm_config_cache; const v = process.env.npm_package_version_number_x;', + 'from huggingface_hub import hf_hub_download, hf_hub_url', + 'secret_key_base: use a long value; secret_santa_gift_exchange_names_for_the_office', + 'pip install scikit-learn sk-learn skeleton-key-ring ASK-THE-TEAM', + 'ATATT and ATBB are the token prefixes Atlassian uses.', + 'A DSN looks like https://public@sentry.example.com/1 in the docs.', + 'DD_API_KEY is set from the environment; DD-API-KEY: ', + 'Endpoint=sb://x.servicebus.windows.net/;SharedAccessKeyName=root;SharedAccessKey=', + 'https://acct.blob.core.windows.net/c/b?sv=2022-11-02&sp=r&sig=', + 'HEROKU_API_KEY=', + ])('ordinary prose and code pass: %s', (text) => { + for (const file of ['a.md', 'a.js', 'a.json', 'docs/setup.txt']) { + const body = file === 'a.json' ? JSON.stringify({ note: text }) : text; + expect(rulesOf(file, `${body}\n`).filter((rule) => PROVIDER_TOKENS.some(([known]) => known === rule)), file).toEqual([]); + } + }); + + it('the repository tree has no provider-token false positive', SLOW, () => { + const result = spawnSync(process.execPath, [SCANNER], { cwd: REPO_ROOT, encoding: 'utf8', timeout: SLOW_TEST_MS }); + expect(result.status, result.stderr).toBe(0); + }); + + it('scans a megabyte of provider prefixes in linear time', SLOW, () => { + // Catastrophic backtracking takes minutes on these inputs; the limit is generous because a few of the generic rules + // (AccountKey=, DD_API_KEY:) legitimately take seconds on a megabyte of one repeated prefix, more on a busy runner. + const timings = timeInChild(` + const prefixes = ${JSON.stringify(ALL_HOSTILE_PREFIXES)}; + for (const prefix of prefixes) { + const text = prefix.repeat(Math.ceil((1024 * 1024) / prefix.length)); + const started = performance.now(); + scanText('a.md', text); + timings.push([prefix, Math.round(performance.now() - started)]); + }`); + for (const [prefix, ms] of timings) expect(ms, prefix).toBeLessThan(PREFIX_FLOOD_LIMIT_MS); + }); + + it('an all-uppercase or single-character token body after every unbounded prefix is linear (was quadratic)', SLOW, () => { + const timings = timeInChild(` + const prefixes = ${JSON.stringify(ALL_HOSTILE_PREFIXES)}; + for (const prefix of prefixes) { + for (const body of ['A', 'ATBB', 'AbC']) { + const text = prefix + body.repeat(Math.ceil((256 * 1024) / body.length)); + for (const file of ['a.md', 'package-lock.json', 'settings.xml']) { + const started = performance.now(); + scanText(file, text); + timings.push([prefix + ' ' + body + ' ' + file, Math.round(performance.now() - started)]); + } + } + }`); + for (const [label, ms] of timings) expect(ms, label).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + describe('(2) --range and the CI workflow', () => { + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + const run = (cmd, args, cwd) => spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: SLOW_TEST_MS }); + const scan = (cwd, ...args) => run(process.execPath, [SCANNER, ...args], cwd); + const git = (cwd, ...args) => run('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', '-c', 'commit.gpgsign=false', ...args], cwd); + const write = (dir, file, content) => { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), content); + }; + const commit = (dir, message, files = {}) => { + for (const [file, content] of Object.entries(files)) write(dir, file, content); + git(dir, 'add', '-A'); + expect(git(dir, 'commit', '-q', '-m', message).status).toBe(0); + return git(dir, 'rev-parse', 'HEAD').stdout.trim(); + }; + const makeRepo = () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-range-')); + dirs.push(dir); + expect(run('git', ['init', '-q', '-b', 'main'], dir).status).toBe(0); + return dir; + }; + const value = randomString(28, 2401); + const leak = `${['API_', 'KEY'].join('')}=${value}\n`; + const clean = `${['API_', 'KEY'].join('')}=your_api_key_here\n`; + const noValue = (result) => { + const output = `${result.stdout}${result.stderr}`; + for (const piece of windows(value)) expect(output).not.toContain(piece); + }; + + it.skipIf(!hasGit())('finds a secret committed and removed again inside the range, which the tip scan cannot see', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, 'base', { 'base.txt': 'base\n' }); + const leaked = commit(dir, 'add key', { 'x.env': leak }); + const head = commit(dir, 'remove key', { 'x.env': clean }); + expect(scan(dir).status).toBe(0); // the reported gap: the tip is clean + const result = scan(dir, '--range', `${base}..${head}`); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${leaked.slice(0, 7)} x.env secret-assignment x1`); + expect(result.stderr).toContain('--range'); + noValue(result); + }); + + it.skipIf(!hasGit())('accepts --range=.. and branch names', SLOW, () => { + const dir = makeRepo(); + commit(dir, 'base', { 'base.txt': 'base\n' }); + git(dir, 'checkout', '-q', '-b', 'feature'); + commit(dir, 'add key', { 'x.env': leak }); + expect(scan(dir, '--range=main..feature').status).toBe(1); + }); + + it.skipIf(!hasGit())('exits 0 on a clean range and on a range without commits', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, 'base', { 'base.txt': 'base\n' }); + const head = commit(dir, 'more', { 'x.env': clean }); + const ok = scan(dir, '--range', `${base}..${head}`); + expect(ok.status).toBe(0); + expect(ok.stdout).toContain('no hits in 1 commit'); + const empty = scan(dir, '--range', `${head}..${head}`); + expect(empty.status).toBe(0); + expect(empty.stdout).toContain('no commits'); + const behind = scan(dir, '--range', `${head}..${base}`); // head is an ancestor of base: nothing new + expect(behind.status).toBe(0); + }); + + it.skipIf(!hasGit())('reads only the range: a secret in an earlier commit is not blamed, and the walk is as long as the range', SLOW, () => { + const dir = makeRepo(); + commit(dir, 'old leak', { 'old.env': leak }); + commit(dir, 'old fix', { 'old.env': clean }); + for (let i = 0; i < 40; i += 1) commit(dir, `filler ${i}`, { [`f${i}.txt`]: `${i}\n` }); + const base = git(dir, 'rev-parse', 'HEAD').stdout.trim(); + const head = commit(dir, 'new', { 'new.txt': 'new\n' }); + const result = scan(dir, '--range', `${base}..${head}`); + expect(result.status).toBe(0); + expect(result.stdout).toContain('no hits in 1 commit '); + expect(scan(dir, '--history').status).toBe(1); // the full audit still finds the old leak + }); + + it.skipIf(!hasGit())('handles a merge commit inside the range (both sides are scanned, once)', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, 'base', { 'base.txt': 'base\n' }); + git(dir, 'checkout', '-q', '-b', 'feature'); + const leaked = commit(dir, 'add key', { 'x.env': leak }); + git(dir, 'checkout', '-q', 'main'); + commit(dir, 'main moves', { 'main.txt': 'm\n' }); + git(dir, 'checkout', '-q', 'feature'); + expect(git(dir, 'merge', '-q', '--no-ff', '-m', 'merge main', 'main').status).toBe(0); + const head = commit(dir, 'remove key', { 'x.env': clean }); + const result = scan(dir, '--range', `${base}..${head}`); + expect(result.status).toBe(1); + expect(result.stderr.split(`${leaked.slice(0, 7)} x.env`)).toHaveLength(2); // exactly one occurrence + }); + + it.skipIf(!hasGit())('a secret only a merge conflict resolution introduced is found in the range', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, 'base', { 'x.env': 'A=1\n' }); + git(dir, 'checkout', '-q', '-b', 'feature'); + commit(dir, 'feature edit', { 'x.env': 'A=2\n' }); + git(dir, 'checkout', '-q', 'main'); + commit(dir, 'main edit', { 'x.env': 'A=3\n' }); + git(dir, 'checkout', '-q', 'feature'); + expect(git(dir, 'merge', '-q', 'main').status).toBe(1); // a conflict, not a failure to run + write(dir, 'x.env', `A=4\n${leak}`); + git(dir, 'add', '-A'); + expect(git(dir, 'commit', '-q', '-m', 'resolve').status).toBe(0); + const head = git(dir, 'rev-parse', 'HEAD').stdout.trim(); + const result = scan(dir, '--range', `${base}..${head}`); + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env secret-assignment'); + }); + + it.skipIf(!hasGit())('uses merge-base semantics: commits only on the base side are not blamed when the base is not an ancestor', SLOW, () => { + const dir = makeRepo(); + commit(dir, 'root', { 'root.txt': 'r\n' }); + git(dir, 'checkout', '-q', '-b', 'feature'); + const feature = commit(dir, 'feature work', { 'f.txt': 'f\n' }); + git(dir, 'checkout', '-q', 'main'); + const mainLeak = commit(dir, 'main leak', { 'm.env': leak }); + const result = scan(dir, '--range', `${mainLeak}..${feature}`); + expect(result.status).toBe(0); + expect(result.stdout).toContain('no hits in 1 commit'); + }); + + it.skipIf(!hasGit())('sees a PR head that a fork provides only through the merge commit (checkout of refs/pull/N/merge)', SLOW, () => { + const upstream = makeRepo(); + const base = commit(upstream, 'base', { 'base.txt': 'base\n' }); + git(upstream, 'checkout', '-q', '-b', 'fork-work'); + commit(upstream, 'add key', { 'x.env': leak }); + const head = commit(upstream, 'remove key', { 'x.env': clean }); + git(upstream, 'checkout', '-q', 'main'); + git(upstream, 'merge', '-q', '--no-ff', '-m', 'PR merge', head); + git(upstream, 'update-ref', 'refs/pull/1/merge', 'HEAD'); + git(upstream, 'reset', '-q', '--hard', base); + git(upstream, 'branch', '-q', '-D', 'fork-work'); + const runner = mkdtempSync(path.join(tmpdir(), 'check-secrets-range-')); + dirs.push(runner); + expect(run('git', ['clone', '-q', upstream, runner], tmpdir()).status).toBe(0); + expect(git(runner, 'fetch', '-q', 'origin', 'refs/pull/1/merge').status).toBe(0); + git(runner, 'checkout', '-q', 'FETCH_HEAD'); + expect(scan(runner, '--range', `${base}..${head}`).status).toBe(1); + }); + + it.skipIf(!hasGit())('a shallow clone that holds BOTH commits (a depth-1 fetch of each) still fails closed, never "no hits"', SLOW, () => { + const upstream = makeRepo(); + const base = commit(upstream, 'base', { 'base.txt': 'base\n' }); + commit(upstream, 'add key', { 'x.env': leak }); + const head = commit(upstream, 'remove key', { 'x.env': clean }); + const shallow = mkdtempSync(path.join(tmpdir(), 'check-secrets-range-')); + dirs.push(shallow); + expect(run('git', ['init', '-q'], shallow).status).toBe(0); + expect(run('git', ['fetch', '-q', '--depth=1', `file://${upstream}`, head], shallow).status).toBe(0); + expect(run('git', ['fetch', '-q', '--depth=1', `file://${upstream}`, base], shallow).status).toBe(0); + expect(run('git', ['rev-parse', '--is-shallow-repository'], shallow).stdout.trim()).toBe('true'); + expect(run('git', ['cat-file', '-t', base], shallow).stdout.trim()).toBe('commit'); // both ends are present + const result = scan(shallow, '--range', `${base}..${head}`); + expect(result.status).toBe(2); + expect(result.stderr).toContain('shallow'); + expect(result.stdout).not.toContain('no hits'); + }); + + it.skipIf(!hasGit())('rejects malformed range specs before git sees them (exit 2, nothing scanned)', SLOW, () => { + const dir = makeRepo(); + const head = commit(dir, 'base', { 'base.txt': 'base\n' }); + for (const spec of ['--output=/tmp/x..HEAD', `${head}..--all`, `${head}...${head}`, `..${head}`, `${head}..`, 'a..b..c', `${head}`, `${head} ..${head}`, '-x..HEAD', 'HEAD..-x']) { + const result = scan(dir, '--range', spec); + expect(result.status, spec).toBe(2); + expect(result.stderr, spec).toContain('expected ..'); + expect(result.stdout, spec).toBe(''); + } + }); + + it.skipIf(!hasGit())('an unreachable base (force-push, partial fetch) or head fails closed with exit 2 and a message', SLOW, () => { + const dir = makeRepo(); + const head = commit(dir, 'base', { 'base.txt': 'base\n' }); + const missing = '1'.repeat(40); + const noBase = scan(dir, '--range', `${missing}..${head}`); + expect(noBase.status).toBe(2); + expect(noBase.stderr).toContain('INCOMPLETE'); + expect(noBase.stderr).toContain('not in this repository'); + expect(noBase.stdout).not.toContain('no hits'); + const noHead = scan(dir, '--range', `${head}..${missing}`); + expect(noHead.status).toBe(2); + expect(noHead.stderr).toContain('INCOMPLETE'); + }); + + it.skipIf(!hasGit())('a shallow clone fails closed (exit 2), never "no hits"', SLOW, () => { + const upstream = makeRepo(); + const base = commit(upstream, 'base', { 'base.txt': 'base\n' }); + const head = commit(upstream, 'add key', { 'x.env': leak }); + const shallow = mkdtempSync(path.join(tmpdir(), 'check-secrets-range-')); + dirs.push(shallow); + expect(run('git', ['clone', '-q', '--depth', '1', `file://${upstream}`, shallow], tmpdir()).status).toBe(0); + const result = scan(shallow, '--range', `${base}..${head}`); + expect(result.status).toBe(2); + expect(result.stderr).toContain('shallow'); + expect(`${result.stdout}${result.stderr}`).not.toContain('no hits'); + }); + + it.skipIf(!hasGit())('an all-zero base (a new branch) scans the tip commit only, and says so', SLOW, () => { + const dir = makeRepo(); + commit(dir, 'first', { 'old.env': leak }); + const head = commit(dir, 'tip', { 'new.txt': 'n\n' }); + const zero = '0'.repeat(40); + const clean1 = scan(dir, '--range', `${zero}..${head}`); + expect(clean1.status).toBe(0); + expect(clean1.stdout).toContain('tip commit only'); + const leakyTip = commit(dir, 'leaky tip', { 'y.env': leak }); + expect(scan(dir, '--range', `${zero}..${leakyTip}`).status).toBe(1); + // The root commit has no parent: still scanned. + const root = git(dir, 'rev-list', '--max-parents=0', 'HEAD').stdout.trim(); + expect(scan(dir, '--range', `${zero}..${root}`).status).toBe(1); + }); + + it.skipIf(!hasGit())('rejects malformed and option-like ranges (exit 2) and never runs shell text', SLOW, () => { + const dir = makeRepo(); + const head = commit(dir, 'base', { 'base.txt': 'base\n' }); + for (const spec of [`${head}...${head}`, `-x..${head}`, `${head}..--output=x`, `${head}`, '..', `..${head}`, `${head}..`, `a b..${head}`]) { + const result = scan(dir, '--range', spec); + expect(result.status, spec).toBe(2); + } + expect(scan(dir, '--range').status).toBe(2); + expect(scan(dir, '--range', `${head}..${head}`, '--history').status).toBe(2); + expect(scan(dir, '--range', `${head}..${head}`, '--range', `${head}..${head}`).status).toBe(2); + const inject = scan(dir, '--range', `$(touch pwned)..${head}`); + expect(inject.status).toBe(2); + const injectHead = scan(dir, '--range', `${head}..\`touch pwned\``); + expect(injectHead.status).toBe(2); + expect(run('ls', [], dir).stdout).not.toContain('pwned'); + }); + + it.skipIf(!hasGit())('applies the lockfile rules and the split-pair context to added lines', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, 'base', { 'yarn.lock': '# base\n', 'k.yaml': `- name: ${NAME}\n type: secret\n` }); + const url = `${scheme}${value}@registry.internal/x/-/x-1.0.0.tgz`; + const withUrl = commit(dir, 'add dependency', { 'yarn.lock': `# base\n"x@^1":\n version "1.0.0"\n resolved "${url}"\n` }); + commit(dir, 'drop dependency', { 'yarn.lock': '# base\n' }); + const withValue = commit(dir, 'add the value under the unchanged name', { 'k.yaml': `- name: ${NAME}\n value: ${value}\n type: secret\n` }); + commit(dir, 'remove the value', { 'k.yaml': `- name: ${NAME}\n value: your_value_here\n type: secret\n` }); + const result = scan(dir, '--range', `${base}..HEAD`); + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${withUrl.slice(0, 7)} yarn.lock lockfile-credential`); + expect(result.stderr).toContain(`${withValue.slice(0, 7)} k.yaml`); + noValue(result); + }); + + it.skipIf(!hasGit())('an ordinary lockfile change in the range passes', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, 'base', { 'package-lock.json': '{}\n' }); + const head = commit(dir, 'update deps', { 'package-lock.json': `${JSON.stringify({ packages: { 'node_modules/a': { resolved: `${scheme}registry.npmjs.org/a/-/a-1.0.0.tgz`, integrity: sha512(2402) } } }, null, 2)}\n` }); + expect(scan(dir, '--range', `${base}..${head}`).status).toBe(0); + }); + + it.skipIf(!hasGit())('an added file version too large to scan makes the range INCOMPLETE (exit 2)', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, 'base', { 'base.txt': 'base\n' }); + const head = commit(dir, 'big', { 'big.json': `${`${'x'.repeat(1023)}\n`.repeat(5 * 1024 + 8)}` }); + const result = scan(dir, '--range', `${base}..${head}`); + expect(result.status).toBe(2); + expect(result.stderr).toContain('INCOMPLETE'); + expect(result.stdout).not.toContain('no hits'); + }); + + it.skipIf(!hasGit())('a lockfile over 5 MB but under the lockfile limit is scanned, not failed', SLOW, () => { + const dir = makeRepo(); + write(dir, 'package-lock.json', `${`{"a":"${'x'.repeat(1000)}"}\n`.repeat(6 * 1024)}`); + git(dir, 'add', '-A'); + const result = scan(dir); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('1 files scanned'); + }); + + it.skipIf(!hasGit())('a credential in a tracked lockfile fails the tree scan and prints only path, line and rule', SLOW, () => { + const dir = makeRepo(); + write(dir, 'package-lock.json', `{\n "resolved": "${scheme}user:${value}@registry.internal/x.tgz"\n}\n`); + git(dir, 'add', '-A'); + const result = scan(dir); + expect(result.status).toBe(1); + expect(result.stderr).toContain('package-lock.json:2 url-password'); + noValue(result); + }); + + // The workflow is a security control, so its shape is tested: the range step, full history, least privilege, and no + // event value interpolated into a shell script. + describe('the CI workflow', () => { + const workflow = readFileSync(path.join(REPO_ROOT, '.github/workflows/ci.yml'), 'utf8'); + const jobBlock = workflow.slice(workflow.indexOf(' secret-scan:')); + const runScripts = () => { + const lines = workflow.split('\n'); + const scripts = []; + for (let i = 0; i < lines.length; i += 1) { + const match = /^(\s*)(?:- )?run:\s*(.*)$/.exec(lines[i]); + if (!match) continue; + const indent = match[1].length; + let body = match[2]; + if (body === '|' || body === '>') { + body = ''; + for (let j = i + 1; j < lines.length && (lines[j].trim() === '' || lines[j].search(/\S/) > indent); j += 1) body += `${lines[j]}\n`; + } + scripts.push(body); + } + return scripts; + }; + + it('scans the full history range in addition to the tip', () => { + expect(jobBlock).toContain('fetch-depth: 0'); + expect(jobBlock).toContain('node scripts/check-secrets.mjs\n'); + expect(jobBlock).toContain('node scripts/check-secrets.mjs --range "$base..$head"'); + expect(jobBlock).toContain('github.event.pull_request.base.sha'); + expect(jobBlock).toContain('github.event.pull_request.head.sha'); + expect(jobBlock).toContain('github.event.before'); + }); + it('never runs the full-history audit in CI', () => { + expect(runScripts().join('\n')).not.toContain('--history'); + }); + it('passes event values through env vars and never interpolates an expression into a run script', () => { + for (const script of runScripts()) expect(script, script).not.toContain('${{'); + expect(jobBlock).toContain('PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}'); + }); + it('refuses an event value that is not a plain commit id (exit 2) before it reaches the scanner', () => { + const step = jobBlock.slice(jobBlock.indexOf('Scan commits in this change')); + expect(step).toContain("id_pattern='^([0-9a-f]{40}|[0-9a-f]{64})$'"); + expect(step).toMatch(/if ! \[\[ "\$base" =~ \$id_pattern && "\$head" =~ \$id_pattern \]\]; then[\s\S]*?exit 2\s+fi/); + expect(step.indexOf('id_pattern')).toBeLessThan(step.indexOf('node scripts/check-secrets.mjs --range')); + }); + it('keeps least privilege and does not use pull_request_target', () => { + expect(jobBlock).toMatch(/permissions:\n\s+contents: read/); + expect(workflow).not.toContain('pull_request_target'); + expect(jobBlock).toContain('persist-credentials: false'); + }); + it('is valid enough to parse: every "run:" block is well-formed and the env names the script reads are all defined', () => { + const step = jobBlock.slice(jobBlock.indexOf('Scan commits in this change')); + for (const name of ['EVENT_NAME', 'PR_BASE_SHA', 'PR_HEAD_SHA', 'PUSH_BEFORE_SHA', 'PUSH_AFTER_SHA']) { + expect(step.split(`${name}: `), name).toHaveLength(2); // exactly one occurrence + expect(step.includes(`"$${name}"`), name).toBe(true); + } + }); + }); + }); +}); + +describe('review round 10', () => { + const rulesOf = (file, text) => scanText(file, text).map((f) => f.rule); + const PASSWORD = ['pass', 'word'].join(''); + const NAME = ['JWT_', 'SECRET'].join(''); + const passphrase = 'correct horse battery staple'; + const random = randomString(24, 3001); + const scheme = ['https', '://'].join(''); + + describe('lockfile auth fields are judged by the shape of the whole value, not its first character', () => { + const LOCK_SHAPES = [ + ['package-lock.json password', 'package-lock.json', (v) => `{"a":{"version":"1.0.0","${PASSWORD}":"${v}"}}\n`], + ['npm-shrinkwrap.json password', 'npm-shrinkwrap.json', (v) => `{"a":{"version":"1.0.0","${PASSWORD}":"${v}"}}\n`], + ['yarn.lock _authToken', 'yarn.lock', (v) => `//registry.corp.net/:_authToken=${v}\n`], + ['pnpm-lock.yaml token', 'pnpm-lock.yaml', (v) => `settings:\n ${['to', 'ken'].join('')}: ${v}\n`], + ['Pipfile.lock _password', 'Pipfile.lock', (v) => `{"_meta":{"sources":[{"_${PASSWORD}":"${v}"}]}}\n`], + ]; + describe.each(LOCK_SHAPES)('%s', (_label, file, shape) => { + it.each(['x', 'X', 'v', 'V', '3', '0', '9', 'q', 'K', '^', '~', '*', '='])('a credential starting with %s is found', (first) => { + for (let i = 0; i < 6; i += 1) { + const value = first + randomString(28, 3100 + i * 7 + first.charCodeAt(0)); + expect(rulesOf(file, shape(value)), `${file} ${first}${i}`).toContain('lockfile-credential'); + } + }); + }); + + it.each(['1.0.0', '^1.2.3', '~1.2', '>=1.0.0 <2.0.0', '^1.0.0 || ^2.0.0', '1.x', '1.2.x', 'x', 'X', '*', '^*', 'v2.0.0', '3', '1.0.0-beta.2', '1.0.0-rc1', '1.0.0 - 2.0.0', 'latest', 'next', 'npm:other@1', 'workspace:*', 'link:../x', 'file:../x', './local', '../local', '/abs/path/pkg', 'true', 'catalog:'])('the dependency specifier %s is not a credential', (spec) => { + expect(scanText('package-lock.json', `{"a":{"${PASSWORD}":"${spec}"}}\n`), spec).toEqual([]); + expect(scanText('yarn.lock', `${PASSWORD} "${spec}"\n`), spec).toEqual([]); + }); + + it('prose in free-text package metadata is not an auth field', () => { + for (const [file, text] of [ + ['composer.lock', '"description": "CSRF token: generation and validation for forms",\n'], + ['composer.lock', '"description": "Store a secret: encrypt at rest",\n'], + ['poetry.lock', 'description = "Utilities for token: parsing and secret: rotation"\n'], + ['package-lock.json', '"description": "Token: bucket rate limiter with password: optional",\n'], + ['yarn.lock', ' summary "The secret: to a good token: cache"\n'], + ]) { + expect(scanText(file, text), text).toEqual([]); + } + }); + + it('a cache-key or sort-key query parameter is an identifier; a credential-qualified key is not', () => { + const url = (query) => `{"resolved":"${scheme}r.example.org/a.tgz?${query}"}\n`; + expect(scanText('package-lock.json', url('cache-key=1234567890abcdef'))).toEqual([]); + expect(scanText('package-lock.json', url('sort-key=1234567890abcdef'))).toEqual([]); + for (const name of ['api_key', 'access-key', 'sig', 'token']) { + expect(rulesOf('package-lock.json', url(`${name}=${random}`)), name).toContain('lockfile-credential'); + } + }); + + it('a provider token hidden in an integrity digest is ignored only because digests are blanked', () => { + // "+" before the prefix satisfies the token's boundary. The same text in a non-lockfile is a finding. + const digest = `sha512-${randomString(51, 3201, ALNUM)}+${['AT', 'BB'].join('')}${randomString(30, 3202, ALNUM)}==`; + const line = `{"packages":{"node_modules/x":{"version":"1.0.0","integrity":"${digest}"}}}\n`; + expect(scanText('package-lock.json', line)).toEqual([]); + expect(rulesOf('data.json', line)).toContain('atlassian-token'); + }); + }); + + describe('XML: namespaced, CDATA and attribute-named elements', () => { + const CONFIG_XML = [ + 'settings.xml', 'pom.xml', 'conf/server.xml', 'web.config', 'app.config', 'App.csproj', 'App.vcxproj', 'x.props', 'x.targets', + 'Info.plist', 'Strings.resx', 'service.wsdl', 'strings.xml', 'gradle.properties.xml', 'App.pubxml', + ]; + const ELEMENTS = [ + ['CDATA', (v) => `<${PASSWORD}>`], + ['CDATA with margins', (v) => `<${PASSWORD}>\n \n`], + ['WS-Security PasswordText', (v) => `${v}`], + ['namespace prefix with a declaration', (v) => `${v}`], + ['prefixed CDATA', (v) => ``], + ['entry key attribute', (v) => `${v}`], + ['item name attribute', (v) => `${v}`], + ['env name attribute', (v) => `${v}`], + ['entry key attribute, single quotes, CDATA', (v) => ``], + ]; + describe.each(CONFIG_XML)('in %s', (file) => { + it.each(ELEMENTS)('a random value is found: %s', (_label, shape) => { + expect(scanText(file, `${shape(random)}\n`).length).toBeGreaterThan(0); + }); + it.each(ELEMENTS)('a passphrase with spaces is found: %s', (_label, shape) => { + expect(scanText(file, `${shape(passphrase)}\n`).length).toBeGreaterThan(0); + }); + }); + + it.each([ + ['a placeholder in CDATA', `<${PASSWORD}>`], + ['an environment reference in CDATA', `<${PASSWORD}>`], + ['an empty prefixed element', ``], + ['a non-secret attribute-named entry', `${random}`], + ['mismatched closing tag', `<${PASSWORD}>`], + ['a CDATA section that never closes', `<${PASSWORD}>`], + ])('%s is clean', (_label, text) => { + expect(scanText('settings.xml', `${text}\n`)).toEqual([]); + }); + }); + + describe('XML-family files outside the first extension list, and template or backup suffixes', () => { + const CONFIG_EXTENSIONS_AND_NAMES = [ + 'App.vcxproj', 'App.sqlproj', 'App.wixproj', 'build.proj', 'App.ccproj', 'App.dcproj', 'App.jsproj', 'App.projitems', 'ServiceConfiguration.cscfg', + 'ServiceDefinition.csdef', 'plan.jmx', 'wifi.mobileconfig', 'app.entitlements', 'module.iml', 'run.launch', 'rules.ruleset', + 'Package.appxmanifest', 'setup.wxs', 'lib-1.0.pom', 'app.jnlp', + 'settings.xml.template', 'settings.xml.dist', 'settings.xml.sample', 'settings.xml.example', 'settings.xml.erb', 'settings.xml.j2', + 'settings.xml.jinja2', 'settings.xml.tpl', 'settings.xml.tmpl', 'settings.xml.bak', 'settings.xml.orig', 'settings.xml.default', 'settings.xml.in', + 'web.config.template', 'App.csproj.erb', 'conf/server.xml.dist.bak', + ]; + it.each(CONFIG_EXTENSIONS_AND_NAMES)('%s is XML configuration', (file) => { + expect(isXmlConfigPath(file)).toBe(true); + expect(fileMode(file)).toBe('config'); + for (const shape of [`<${PASSWORD}>${random}`, `<${PASSWORD}>${passphrase}`, ``]) { + expect(scanText(file, `${shape}\n`).length, `${file}: ${shape}`).toBeGreaterThan(0); + } + }); + + it('the plist-like Apple profile form PasswordV is found', () => { + expect(scanText('wifi.mobileconfig', `${['Pass', 'word'].join('')}${random}\n`).length).toBeGreaterThan(0); + }); + + it.each(['icon.svg', 'index.html', 'page.htm', 'page.xhtml', 'style.xsl', 'style.xslt', 'View.xaml', 'schema.xsd', 'feed.rss', 'feed.atom', 'map.kml'])('%s stays in code mode (markup and schema, not settings): a form label or a passphrase-like text is not noisy', (file) => { + expect(isXmlConfigPath(file)).toBe(false); + expect(fileMode(file)).toBe('code'); + const text = `<${PASSWORD}>${passphrase}\n\n\n`; + expect(scanText(file, text), file).toEqual([]); + }); + + it.each(['notes.md', 'app.js', 'app.js.template', 'x.py.bak'])('%s is not XML configuration', (file) => { + expect(isXmlConfigPath(file)).toBe(false); + }); + + it('a suffix-only name is not stripped to nothing', () => { + expect(isXmlConfigPath('.template')).toBe(false); + expect(isXmlConfigPath('.bak')).toBe(false); + }); + }); + + describe('common non-secret XML is not noisy', () => { + it.each([ + ['an Android Maps key as a resource reference', 'AndroidManifest.xml', ''], + ['the same tag split over three lines', 'AndroidManifest.xml', ''], + ['a color and an attr reference', 'app/src/main/res/values/styles.xml', '@color/red?attr/colorPrimary'], + ['an @token@ substitution', 'build.xml', `<${PASSWORD}>@${PASSWORD}@`], + ['a #{token} substitution', 'pom.xml', `<${PASSWORD}>#{${PASSWORD}}`], + ['a %%TOKEN%% substitution', 'settings.xml', `<${PASSWORD}>%%${PASSWORD.toUpperCase()}%%`], + ['a D-Bus interface', 'org.example.Home.xml', '\n \n\n\n \n'], + ['a D-Bus interface with an entry-like tag before', 'org.example.Home.xml', ''], + ['Maven reference example blocks', 'conf/settings.xml', `example-proxyproxy.example.comproxyuser<${PASSWORD}>proxypass\nsiteServer/path/to/private/keyoptional; leave empty if not used.`], + ])('%s is clean', (_label, file, text) => { + expect(scanText(file, `${text}\n`), text).toEqual([]); + }); + + it('a real value is still found next to those examples', () => { + expect(scanText('AndroidManifest.xml', `\n`).length).toBeGreaterThan(0); + expect(scanText('settings.xml', `${random}\n`).length).toBeGreaterThan(0); + expect(scanText('settings.xml', `<${PASSWORD}>${passphrase}\n`).length).toBeGreaterThan(0); + expect(scanText('settings.xml', `\n\n`).length).toBeGreaterThan(0); + }); + + // Class: sentence punctuation is not a documentation cue. A passphrase of plain words stays a finding whatever ends it, in + // XML (element text, CDATA, a key= entry) and in every other format; only a documentation cue (or a message catalog) exempts. + const ENDINGS = ['', '!', '.', '?', ',', ';', ':', '...', '\u2026', '!!', '"', ')', ' :)', '\u3002', '\uFF01', '\u{1F600}']; + const plainPhrase = ['tulip', 'marble', 'sunset', 'harbor'].join(' '); + const ARRANGEMENTS = [ + ['an XML element', 'settings.xml', (v) => `<${PASSWORD}>${v}\n`], + ['an XML CDATA element', 'settings.xml', (v) => `<${PASSWORD}>\n`], + ['an XML key entry', 'app.config', (v) => `${v}\n`], + ['an XML attribute', 'app.config', (v) => `\n`], + ['a quoted YAML value', 'config.yml', (v) => `${PASSWORD}: "${v}"\n`], + ['a bare YAML value', 'config.yml', (v) => `${PASSWORD}: ${v}\n`], + ['an ini value', 'config.ini', (v) => `${PASSWORD} = ${v}\n`], + ['a properties value', 'config.properties', (v) => `db.${PASSWORD}=${v}\n`], + ['a JSON value', 'config.json', (v) => `{"${PASSWORD}": "${v}"}\n`], + ['a TOML value', 'config.toml', (v) => `${PASSWORD} = "${v}"\n`], + ['a quoted env value', '.env', (v) => `DB_${PASSWORD.toUpperCase()}="${v}"\n`], + ['a bare env value', '.env', (v) => `DB_${PASSWORD.toUpperCase()}=${v}\n`], + ['a CSV cell', 'export.csv', (v) => `name,value\n${PASSWORD},"${v}"\n`], + ['a Markdown table cell', 'notes.md', (v) => `| Name | Value |\n|---|---|\n| DB_${PASSWORD.toUpperCase()} | \`${v}\` |\n`], + ]; + describe.each(ARRANGEMENTS)('%s', (_label, file, make) => { + it.each(ENDINGS)(`a passphrase of plain words ending in %j is found`, (ending) => { + expect(scanText(file, make(`${plainPhrase}${ending}`)).length).toBeGreaterThan(0); + }); + }); + + it.each([ + 'the password you chose during setup.', + 'Enter your password.', + 'Ask your team lead for the password.', + 'Set via environment variable at runtime.', + ])('documentation prose %j stays exempt in XML, as it does elsewhere', (prose) => { + expect(scanText('settings.xml', `<${PASSWORD}>${prose}\n`)).toEqual([]); + expect(scanText('config.yml', `${PASSWORD}: "${prose}"\n`)).toEqual([]); + }); + + it('a sentence with a documentation cue plus a random word is still found', () => { + expect(scanText('settings.xml', `<${PASSWORD}>the password you chose is ${random}!\n`).length).toBeGreaterThan(0); + }); + + it('a message catalog keeps its exemption for a punctuated sentence, and the same text elsewhere is found', () => { + const text = `This is the way!\n`; + expect(scanText('app/src/main/res/values/auth.xml', text)).toEqual([]); + expect(scanText('settings.xml', text).length).toBeGreaterThan(0); + expect(scanText('i18n/en.json', `{"${PASSWORD}": "${plainPhrase}\u2026"}\n`)).toEqual([]); + }); + + it('a trailing ellipsis still marks a cut-off token as a placeholder', () => { + expect(scanText('config.yml', `${PASSWORD}: "Bearer ${random.slice(0, 12)}..."\n`)).toEqual([]); + expect(scanText('config.yml', `${PASSWORD}: "${random.slice(0, 12)}..."\n`)).toEqual([]); + }); + + it('an Android res/values sentence under a password-like name is a message, but the same text elsewhere is a passphrase', () => { + // Not strings.xml: that name is already a message catalog by its file name, so only the res/values directory rule counts here. + const text = `This is the way\n`; + expect(scanText('app/src/main/res/values/auth.xml', text)).toEqual([]); + expect(scanText('app/src/main/res/values-fr/auth.xml', text)).toEqual([]); + expect(scanText('settings.xml', text).length).toBeGreaterThan(0); + }); + }); + + describe('provider tokens: Slack, SendGrid and Sentry shapes', () => { + const digits = (n, seed) => randomString(n, seed, DIGITS); + it.each([ + 'xoxo-love-and-kisses-2026', + 'xoxo-team-2026-Q1-kickoff-notes', + 'xoxe-1-planning-2026-notes-for-the-quarter-review-meeting-agenda-items-list', + `${['xapp', '-1-A0000000000-0000000000000-'].join('')}${'0'.repeat(64)}`, + `${['xox', 'b-000000000000-000000000000-'].join('')}${'x'.repeat(24)}`, + `${['xox', 'o-'].join('')}${'0'.repeat(12)}-${'0'.repeat(12)}-${'0'.repeat(12)}-${'0'.repeat(32)}`, + `${['S', 'G.'].join('')}${'x'.repeat(22)}.${'y'.repeat(43)}`, + `${['S', 'G.'].join('')}${'a'.repeat(22)}.${'b'.repeat(43)}`, + ])('%s is not a token', (text) => { + for (const file of ['a.md', 'a.json', 'a.js']) expect(scanText(file, `see ${text} here\n`), file).toEqual([]); + }); + + it('the documented xoxo and xoxe shapes are found', () => { + const xoxo = `${['xox', 'o-'].join('')}${digits(12, 3301)}-${digits(12, 3302)}-${digits(12, 3303)}-${randomString(32, 3304, HEX)}`; + const xoxe = `${['xox', 'e-1-'].join('')}${randomString(100, 3305, `${ALNUM}_-`)}`; + for (const token of [xoxo, xoxe]) for (const file of ['a.md', 'a.json', 'a.js']) expect(rulesOf(file, `see ${token} here\n`)).toContain('slack-token'); + }); + + it('a Sentry DSN is reported only when it carries the deprecated secret half', () => { + const key = randomString(32, 3401, HEX); + const secret = randomString(32, 3402, HEX); + const publicDsn = `${scheme}${key}@o123.ingest.sentry.io/456`; + const secretDsn = `${scheme}${key}:${secret}@o123.ingest.sentry.io/456`; + for (const file of ['a.js', 'a.json', 'a.md']) { + expect(rulesOf(file, `Sentry.init({ dsn: '${publicDsn}' });\n`), file).not.toContain('sentry-token'); + expect(rulesOf(file, `Sentry.init({ dsn: '${secretDsn}' });\n`), file).toContain('sentry-token'); + } + }); + }); + + describe('placeholders for the added provider families pass', () => { + const pad = (prefix, n, ch = 'x') => `${prefix}${ch.repeat(n)}`; + it.each([ + pad('123456789:A' + 'A', 33), + pad(['sk', '.eyJ'].join(''), 40) + '.' + 'x'.repeat(22), + pad(['sq0', 'atp-'].join(''), 22), + pad(['sq0', 'csp-'].join(''), 43), + pad(['NR', 'AK-'].join(''), 27, 'X'), + pad(['cf', 'ut_'].join(''), 48), + pad(['xox', 'e-1-'].join(''), 100, '0'), + `M${'x'.repeat(24)}.${'x'.repeat(6)}.${'x'.repeat(30)}`, + ])('%s... passes', (value) => { + for (const [file, text] of [['a.json', `{"note":"${value}"}\n`], ['a.md', `token ${value}\n`], ['a.js', `const t = "${value}";\n`]]) { + expect(rulesOf(file, text), file).toEqual([]); + } + }); + }); + + describe('the quadratic all-uppercase body is gone', () => { + it('nameWords-driven placeholder checks stay fast on a long all-uppercase token body (killable child)', SLOW, () => { + const timings = timeInChild(` + for (const [label, file, text] of [ + ['ATBB x4096', 'package-lock.json', 'ATBB'.repeat(16 * 256)], + ['ATBB x64k', 'a.md', 'ATBB'.repeat(16 * 1024)], + ['glpat A 64k', 'a.md', 'glpat-' + 'A'.repeat(64 * 1024)], + ['sk-ant A 64k', 'settings.xml', 'sk-ant-' + 'A'.repeat(64 * 1024)], + ['upper name 64k', 'a.env', 'API_' + 'KEY'.repeat(20000) + '=x'], + ]) { + const started = performance.now(); + scanText(file, text); + timings.push([label, Math.round(performance.now() - started)]); + }`); + for (const [label, ms] of timings) expect(ms, label).toBeLessThan(2000); + }); + }); +}); + +// --------------------------------------------------------------------------- +// Audit round: command-start anchors, unmerged index modes, markers on separated value lines, whole curl -u arguments. +// Every credential below is fake and built at runtime, so no literal in this file is secret-shaped. +// --------------------------------------------------------------------------- +describe('secret-cli-command: the value source may sit anywhere before the pipe', () => { + const NAME = secretName('JWT_', 'SECRET'); + const value = randomString(24, 8101); + const rules = (text, file = 'deploy.sh') => scanText(file, text).map((f) => f.rule); + + const positives = [ + ['indented echo', ` echo ${value} | vercel env add ${NAME} production\n`], + ['tab-indented echo', `\techo ${value} | vercel env add ${NAME}\n`], + ['CRLF line ending', ` echo ${value} | vercel env add ${NAME}\r\n`], + ['YAML run block', `jobs:\n deploy:\n steps:\n - run: |\n echo ${value} | vercel env add ${NAME}\n`], + ['YAML inline run', ` - run: echo ${value} | vercel env add ${NAME}\n`], + ['shell function body', `deploy() {\n echo ${value} | vercel env add ${NAME}\n}\n`], + ['after &&', `cd app && echo ${value} | vercel env add ${NAME}\n`], + ['after ||', `cd app || echo ${value} | vercel env add ${NAME}\n`], + ['after a pipe', `cat x | echo ${value} | vercel env add ${NAME}\n`], + ['subshell', `(echo ${value} | vercel env add ${NAME})\n`], + ['command substitution', `$(echo ${value} | vercel env add ${NAME})\n`], + ['then', `if x; then echo ${value} | vercel env add ${NAME}; fi\n`], + ['then on its own line', ` then echo ${value} | vercel env add ${NAME}\n`], + ['do', `for e in a b; do echo ${value} | vercel env add ${NAME}; done\n`], + ['else', `else echo ${value} | vercel env add ${NAME}\n`], + ['brace group', `{ echo ${value} | vercel env add ${NAME}; }\n`], + ['negation', `! echo ${value} | vercel env add ${NAME}\n`], + ['time prefix', `time echo ${value} | vercel env add ${NAME}\n`], + ['prompt prefix', `$ echo ${value} | vercel env add ${NAME}\n`], + ['markdown list item', `- echo ${value} | vercel env add ${NAME}\n`], + ['/bin/echo', `/bin/echo ${value} | vercel env add ${NAME}\n`], + ['command echo', `command echo ${value} | vercel env add ${NAME}\n`], + ['echo -n', ` echo -n ${value} | vercel env add ${NAME}\n`], + ['echo -n quoted', ` echo -n "${value}" | vercel env add ${NAME}\n`], + ['no spaces around the pipe', `true &&echo ${value}|vercel env add ${NAME}\n`], + ['sudo on the CLI', `echo ${value} | sudo vercel env add ${NAME}\n`], + ['env prefix on the CLI', `echo ${value} | env FOO=1 vercel env add ${NAME}\n`], + ['bash -c', `bash -c "echo ${value} | vercel env add ${NAME}"\n`], + ['sh -c single quotes', `sh -c 'echo ${value} | vercel env add ${NAME}'\n`], + ['value-preserving filter', ` echo ${value} | tr -d '\\n' | vercel env add ${NAME}\n`], + ['printf %s', ` printf '%s' "${value}" | vercel env add ${NAME}\n`], + ['printf %s\\n', ` printf '%s\\n' ${value} | vercel env add ${NAME}\n`], + ['printf with the value as format', ` printf ${value} | vercel env add ${NAME}\n`], + ['pipe at the end of the previous line', ` echo ${value} |\n vercel env add ${NAME}\n`], + ['backslash continuation', ` echo ${value} | \\\n sudo vercel env add ${NAME}\n`], + ['gh secret set', ` echo ${value} | gh secret set ${NAME}\n`], + ['netlify positional', ` echo ${value} | netlify env:set ${NAME}\n`], + ['here-string, indented', ` vercel env add ${NAME} <<< ${value}\n`], + ['here-string after sudo', ` sudo vercel env add ${NAME} <<< "${value}"\n`], + ['cat heredoc into the CLI', `cat <<'EOF' | vercel env add ${NAME}\n${value}\nEOF\n`], + ['indented cat heredoc', ` cat < { + expect(rules(text)).toContain('secret-cli-command'); + }); + + const negatives = [ + ['placeholder value', ` echo your_jwt_secret_here | vercel env add ${NAME}\n`], + ['environment reference', ` echo "$JWT_VALUE" | vercel env add ${NAME}\n`], + ['braced reference', ` echo "\${JWT_VALUE}" | vercel env add ${NAME}\n`], + ['non-secret variable name', ` echo ${value} | vercel env add PUBLIC_URL\n`], + ['echo feeds a different command, then a separate one runs', `echo ${value} | tee log; vercel env add ${NAME} { + expect(rules(text)).toEqual([]); + }); + + it('stays fast on a long line of echo words', SLOW, () => { + const started = performance.now(); + scanText('a.sh', `${'echo | '.repeat(20000)}vercel env add ${NAME} x\n`); + scanText('a.sh', `${'echo '.repeat(20000)}| vercel env add ${NAME} x\n`); + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); +}); + +describe('unmerged index entries: only a gitlink stage is skipped', () => { + const NAME = secretName('JWT_', 'SECRET'); + const value = randomString(28, 8203); + const gitlinkId = 'a'.repeat(40); + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + + const run = (cmd, args, cwd, input) => spawnSync(cmd, args, { cwd, encoding: 'utf8', input, timeout: SLOW_TEST_MS }); + + /** stages: [stage, mode, content | null]; a null content is a gitlink. `working`: what the working tree holds. */ + function scenario(stages, working) { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-')); + dirs.push(dir); + expect(run('git', ['init', '-q'], dir).status).toBe(0); + const lines = stages.map(([stage, mode, content]) => { + const id = content === null ? gitlinkId : run('git', ['hash-object', '-w', '--stdin'], dir, content).stdout.trim(); + return `${mode} ${id} ${stage}\tapp.env\n`; + }); + expect(run('git', ['update-index', '--index-info'], dir, lines.join('')).status).toBe(0); + if (working === 'file') writeFileSync(path.join(dir, 'app.env'), `${NAME}=changeme\n`); + if (working === 'symlink') symlinkSync('target.txt', path.join(dir, 'app.env')); + if (working === 'directory') { + mkdirSync(path.join(dir, 'app.env')); + writeFileSync(path.join(dir, 'app.env', 'x.txt'), 'x\n'); + } + const result = run(process.execPath, [SCANNER], dir); + return { result, output: `${result.stdout}\n${result.stderr}` }; + } + + const leaky = `${NAME}=${value}\n`; + const clean = `${NAME}=changeme\n`; + const cases = [ + ['gitlink stage 1 hides a regular stage 2', [[1, '160000', null], [2, '100644', leaky]], 'none'], + ['... with a regular working file', [[1, '160000', null], [2, '100644', leaky]], 'file'], + ['... with a directory in the working tree', [[1, '160000', null], [2, '100644', leaky]], 'directory'], + ['gitlink stage 2, executable stage 3', [[2, '160000', null], [3, '100755', leaky]], 'none'], + ['gitlink stage 3 after a leaky stage 2', [[2, '100644', leaky], [3, '160000', null]], 'file'], + ['symlink stage 1, regular stage 2', [[1, '120000', 'target.txt'], [2, '100644', leaky]], 'file'], + ['symlink stage 1, regular stage 2, nothing in the working tree', [[1, '120000', 'target.txt'], [2, '100644', leaky]], 'none'], + ['executable stage 1, symlink stage 2, symlink in the working tree', [[1, '100755', leaky], [2, '120000', 'target.txt']], 'symlink'], + ['leaky symlink stage 1, clean regular stage 2', [[1, '120000', leaky], [2, '100644', clean]], 'symlink'], + ['three stages, three modes', [[1, '160000', null], [2, '120000', 'target.txt'], [3, '100755', leaky]], 'none'], + ]; + it.skipIf(!hasGit()).each(cases)('%s is reported and leaks nothing', SLOW, (_label, stages, working) => { + const { result, output } = scenario(stages, working); + expect(result.status).toBe(1); + expect(result.stderr).toContain('app.env:1 secret-assignment'); + for (const piece of windows(value)) expect(output).not.toContain(piece); + }); + + it.skipIf(!hasGit())('a path that is a gitlink in every stage is still skipped and counted', SLOW, () => { + const { result } = scenario([[1, '160000', null], [2, '160000', null]], 'none'); + expect(result.status).toBe(0); + expect(result.stdout).toContain('1 skipped: 1 submodule'); + }); + + it.skipIf(!hasGit())('clean stages of mixed modes pass', SLOW, () => { + const { result } = scenario([[1, '100755', clean], [2, '100644', clean]], 'file'); + expect(result.status).toBe(0); + }); +}); + +describe('allow marker on the value line of a separated name/value pair', () => { + const NAME = secretName('JWT_', 'SECRET'); + const value = randomString(26, 8307); + const M = ALLOW_MARKER; + const count = (file, text) => scanText(file, text).length; + + const suppressed = [ + ['YAML list item, adjacent', 'd.yaml', `- name: ${NAME}\n value: ${value} # ${M}\n`], + ['YAML list item, other fields between', 'd.yaml', `- name: ${NAME}\n type: plain\n note: x\n value: ${value} # ${M}\n`], + ['YAML list item, value first', 'd.yaml', `- value: ${value} # ${M}\n type: plain\n name: ${NAME}\n`], + ['Kubernetes env entry', 'd.yaml', `env:\n - name: ${NAME}\n value: ${value} # ${M}\n`], + ['JSON object, value last', 'd.json', `{\n "key": "${NAME}",\n "type": "plain",\n "value": "${value}" // ${M}\n}\n`], + ['JSON object, value first', 'd.json', `{\n "value": "${value}", // ${M}\n "type": "plain",\n "key": "${NAME}"\n}\n`], + ['XML property, value last', 'd.xml', `\n${NAME}\nd\n${value} \n\n`], + ['XML property, value first', 'd.xml', `\n${value} \nd\n${NAME}\n\n`], + ['mapping key with a nested value', 'd.yaml', `${NAME}:\n description: x\n value: ${value} # ${M}\n`], + ['HCL block', 'main.tf', `variable "x" {\n name = "${NAME}"\n description = "d"\n value = "${value}" # ${M}\n}\n`], + ]; + it.each(suppressed)('a marker on the value line suppresses: %s', (_label, file, text) => { + expect(count(file, text)).toBe(0); + // the same document without the marker is a finding (the fixture is real) + expect(count(file, text.replaceAll(M, 'note'))).toBe(1); + }); + + const notSuppressed = [ + ['YAML list item', 'd.yaml', `- name: ${NAME}\n type: plain # ${M}\n value: ${value}\n`], + ['YAML list item, value first', 'd.yaml', `- value: ${value}\n type: plain # ${M}\n name: ${NAME}\n`], + ['JSON object', 'd.json', `{\n "key": "${NAME}",\n "type": "plain", // ${M}\n "value": "${value}"\n}\n`], + ['mapping key with a nested value', 'd.yaml', `${NAME}:\n description: x # ${M}\n value: ${value}\n`], + ['HCL block', 'main.tf', `variable "x" {\n name = "${NAME}"\n description = "d" # ${M}\n value = "${value}"\n}\n`], + ]; + it.each(notSuppressed)('a marker on an unrelated line in between does not suppress: %s', (_label, file, text) => { + expect(count(file, text)).toBe(1); + }); + + it('a marker on one pair does not hide the next pair', () => { + const other = randomString(26, 8309); + const text = `- name: ${NAME}\n value: ${value} # ${M}\n- name: ${secretName('API_', 'TOKEN')}\n value: ${other}\n`; + const found = scanText('d.yaml', text); + expect(found).toHaveLength(1); + expect(found[0].line).toBe(3); + }); + + it('a marker elsewhere in the file does not suppress a finding whose value range is the whole window', () => { + const escaped = `{"a":"{\\"key\\":\\"${NAME}\\",\\"value\\":\\"${value}\\"}",\n"b":"x"}\n# ${M}\n`; + expect(count('d.json', escaped)).toBe(1); + }); +}); + +describe('curl -u and the other password flags judge the whole argument', () => { + const host = 'https://api.internal.corp/v1'; + const first = ['pass', 'word'].join(''); // a placeholder-like first word + const phrase = `${first} ${['correct', 'horse', 'battery'].join(' ')} 123!`; + const token = randomString(20, 8419); + const rules = (text, file = 'run.sh') => scanText(file, text).map((f) => f.rule); + const escapedPhrase = phrase.replaceAll(' ', '\\ '); + + const positives = [ + ['double quotes', `curl -u "admin:${phrase}" ${host}\n`], + ['single quotes', `curl -u 'admin:${phrase}' ${host}\n`], + ['ANSI-C quotes', `curl -u $'admin:${phrase}' ${host}\n`], + ['escaped spaces', `curl -u admin:${escapedPhrase} ${host}\n`], + ['--user', `curl --user "admin:${phrase}" ${host}\n`], + ['--user=', `curl --user="admin:${phrase}" ${host}\n`], + ['-u attached to its quote', `curl -u"admin:${phrase}" ${host}\n`], + ['-u attached to a bare value', `curl -uadmin:${token} ${host}\n`], + ['a flag cluster ending in u', `curl -sSu "admin:${phrase}" ${host}\n`], + ['--proxy-user', `curl --proxy-user "admin:${phrase}" ${host}\n`], + ['--proxy-user, bare', `curl --proxy-user admin:${token} ${host}\n`], + ['-U', `curl -U "admin:${phrase}" ${host}\n`], + ['-U, bare', `curl -U admin:${token} ${host}\n`], + ['a reference plus literal words', `curl -u "admin:\${API_PASS} extra words here 9" ${host}\n`], // check-secrets:allow + ['curl --pass phrase', `curl --cert c.pem --pass "${phrase}" ${host}\n`], + ['wget --password', `wget --user=admin --password "${phrase}" ${host}\n`], + ['wget --password=', `wget --password="${phrase}" ${host}\n`], + ['wget --http-password', `wget --http-password='${phrase}' ${host}\n`], + ['wget --ftp-password', `wget --ftp-password ${token} ftp://h/x\n`], + ['wget --proxy-password', `wget --proxy-password=${token} ${host}\n`], + ['httpie -a', `http -a "admin:${phrase}" ${host}\n`], + ['httpie --auth', `http --auth 'admin:${phrase}' ${host}\n`], + ['httpie -a, bare', `http -a admin:${token} ${host}\n`], + ['https -a', `https -a admin:${token} api.internal.corp/x\n`], + ['xh -a', `xh -a admin:${token} ${host}\n`], + ['httpie bearer token', `http -A bearer -a ${token} ${host}\n`], + ['mysql -p attached', `mysql -u root -p${token} db\n`], + ['mysql -p quoted', `mysql -u root -p'${phrase}' db\n`], + ['mysql --password=', `mysql --password="${phrase}" db\n`], + ['mysqldump -p', `mysqldump -u root -p${token} db\n`], + ['mongosh --password', `mongosh --username u --password "${phrase}"\n`], + ['mongosh -p', `mongosh -u u -p ${token}\n`], + ['redis-cli -a', `redis-cli -a ${token} ping\n`], + ['redis-cli -a quoted', `redis-cli -a "${phrase}" ping\n`], + ['redis-cli --pass', `redis-cli --pass ${token} ping\n`], + ['sshpass -p', `sshpass -p ${token} ssh u@h\n`], + ['sshpass -p quoted', `sshpass -p "${phrase}" ssh u@h\n`], + ['smbclient -U user%password', `smbclient -U 'admin%${token}' //h/s\n`], + ['ldapsearch -w', `ldapsearch -D cn=x -w ${token}\n`], + ]; + it.each(positives)('finds the password: %s', (_label, text) => { + expect(rules(text)).toContain('url-password'); + }); + + const negatives = [ + ['placeholder password', `curl -u admin:${first} ${host}\n`], + ['placeholder in quotes', `curl -u "admin:your_password_here" ${host}\n`], + ['angle-bracket placeholder', `curl -u "admin:" ${host}\n`], + ['environment reference', `curl -u "admin:$API_PASS" ${host}\n`], + ['braced reference', `curl -u "admin:\${API_PASS}" ${host}\n`], + ['user only (prompts)', `curl -u admin ${host}\n`], + ['docker -u uid', 'docker run -u root:root img\n'], + ['documentation about the password', `curl -u "user:the password you chose during setup" ${host}\n`], + ['an unterminated quote belongs to the surrounding string', `x: 'curl -u user:${first}', ${host}\n`], + ['mysql -p prompts', 'mysql -u root -p db\n'], + ['mysql --password without a value', 'mysql --password db\n'], + ['httpie -a placeholder', `http -a user:${first} ${host}\n`], + ['httpie -a without a password', `http -a admin ${host}\n`], + ['redis-cli reference', 'redis-cli -a "$REDIS_PASS" ping\n'], + ['wget reference', 'wget --password="$FTP_PASS" ftp://h/x\n'], + ['sshpass reference', 'sshpass -p "${SSH_PASS}" ssh u@h\n'], + ]; + it.each(negatives)('passes: %s', (_label, text) => { + expect(rules(text)).toEqual([]); + }); + + it('never prints or stores the password, whole or in part', () => { + const found = scanText('run.sh', `curl -u "admin:${phrase}" ${host}\n`); + expect(JSON.stringify(found)).not.toContain('battery'); + expect(Object.keys(found[0]).sort()).toEqual(['line', 'path', 'rule']); + }); + + it('stays fast on hostile flag lines', SLOW, () => { + const started = performance.now(); + for (const text of [ + `curl ${'-u '.repeat(30000)}\n`, + `curl -u${' '.repeat(100000)}x\n`, + `curl -u "${'a:'.repeat(50000)}\n`, + `${'wget --password '.repeat(5000)}\n`, + `${'mysql -p'.repeat(20000)}\n`, + `${'http '.repeat(50000)}-a\n`, + ]) { + scanText('a.sh', text); + } + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); +}); + +// --------------------------------------------------------------------------- +// Review round 11 +// (1) command-style assignments: shell syntaxes that set a variable without "name=value" +// (2) values that span lines: heredocs, triple quotes, template literals, quotes closed later, continuations +// (3) --range / --history: a verified binary is skipped whatever its size; text over the limit still fails closed +// Every fixture is assembled at runtime from pieces (this file holds no secret-shaped literal). +// --------------------------------------------------------------------------- + +describe('review round 11 (1): command-style assignments without "="', () => { + const NAME = ['API_', 'TOKEN'].join(''); + const LOWER_NAME = ['api', 'token'].join('_'); + const PASSWORD_NAME = ['DB_', 'PASSWORD'].join(''); + const value = randomString(24, 8101); + const passphrase = ['correct', 'horse', 'battery', 'staple'].join(' '); + const placeholder = ['your', 'token', 'here'].join('_'); + const count = (file, text) => scanText(file, text).length; + + // [label, file, text]: each is a secret and must be reported by the scanner. + const flagged = [ + // fish: every flag spelling, quoted and unquoted values, several values + ['fish -gx', 'config.fish', `set -gx ${NAME} ${value}\n`], + ['fish long flags and a passphrase', 'config.fish', `set --global --export ${NAME} "${passphrase}"\n`], + ['fish single-quoted passphrase', 'config.fish', `set -x ${NAME} '${passphrase}'\n`], + ['fish -Ux', 'config.fish', `set -Ux ${NAME} ${value}\n`], + ['fish -U', 'config.fish', `set -U ${NAME} ${value}\n`], + ['fish -l', 'config.fish', `set -l ${PASSWORD_NAME} ${value}\n`], + ['fish --universal --export', 'config.fish', `set --universal --export ${NAME} ${value}\n`], + ['fish --local', 'config.fish', `set --local ${NAME} ${value}\n`], + ['fish --append', 'config.fish', `set --append ${NAME} ${value}\n`], + ['fish --prepend', 'config.fish', `set --prepend ${NAME} ${value}\n`], + ['fish without flags', 'config.fish', `set ${NAME} ${value}\n`], + ['fish list written as an unquoted passphrase', 'config.fish', `set -gx ${NAME} ${passphrase}\n`], + ['fish list whose second element is the secret', 'config.fish', `set -gx ${NAME} short ${value}\n`], + ['fish inside fish -c in a shell script', 'setup.sh', `fish -c 'set -gx ${NAME} ${value}'\n`], + ['fish after && on a line', 'setup.sh', `mkdir -p x && set -gx ${NAME} ${value}\n`], + ['fish in a heredoc that a shell script writes', 'setup.sh', `cat > ~/.config/fish/config.fish <<'EOF'\nset -gx ${NAME} ${value}\nEOF\n`], + ['fish in a heredoc in a CI step', 'ci.yml', `steps:\n - run: |\n cat > c.fish < { + const found = scanText(file, text); + expect(found.length).toBeGreaterThanOrEqual(1); + expect(Object.keys(found[0]).sort()).toEqual(['line', 'path', 'rule']); // never the matched text + }); + + const clean = [ + ['fish placeholder', 'config.fish', `set -gx ${NAME} ${placeholder}\n`], + ['fish angle-bracket placeholder', 'config.fish', `set -gx ${NAME} \n`], + ['fish variable reference', 'config.fish', `set -gx ${NAME} $OTHER_VALUE\n`], + ['fish quoted variable reference', 'config.fish', `set -gx ${NAME} "$OTHER_VALUE"\n`], + ['fish command substitution', 'config.fish', `set -gx ${NAME} (cat ~/.token)\n`], + ['fish quoted command substitution', 'config.fish', `set -gx ${NAME} "(cat ~/.token)"\n`], + ['fish erase', 'config.fish', `set -e ${NAME}\n`], + ['fish query', 'config.fish', `set -q ${NAME}\n`], + ['fish show', 'config.fish', `set --show ${NAME}\n`], + ['fish PATH', 'config.fish', 'set -gx PATH $PATH /usr/local/bin\n'], + ['fish weak name with an address', 'config.fish', 'set -gx TOKEN_ENDPOINT https://auth.example.net/oauth/token\n'], + ['fish comment', 'config.fish', `# set -gx ${NAME} ${value}\n`], + ['fish allow marker', 'config.fish', `set -gx ${NAME} ${value} # ${ALLOW_MARKER}\n`], + ['fish SETUVAR placeholder', 'fish_variables', `SETUVAR --export ${NAME}:${placeholder}\n`], + ['fish placeholder in a fence', 'README.md', `\`\`\`fish\nset -gx ${NAME} ${placeholder}\n\`\`\`\n`], + ['the verb "set" in prose outside a fence', 'README.md', `Please set ${LOWER_NAME} ${value} in your shell\n`], + ['"set token" in a sentence', 'README.md', 'We set token verification on the server.\n'], + ['a Python sentence', 'tool.py', 'x = "set up token verification"\n'], + ['a YAML list item that starts with Set', 'ci.yml', '- Set token expiration in the dashboard\n'], + ['a YAML description', 'ci.yml', 'description: Set token expiration in the dashboard\n'], + ['bash set -euo pipefail', 'env.sh', 'set -euo pipefail\nset -x\n'], + ['bash set --', 'env.sh', `set -- "$${NAME}"\n`], + ['bash set +x NAME', 'env.sh', `set +x ${NAME}\n`], + ['csh placeholder', 'env.csh', `setenv ${NAME} ${placeholder}\n`], + ['csh reference', 'env.csh', `setenv ${NAME} $HOME/x\n`], + ['csh PATH', 'env.csh', 'setenv PATH /usr/bin\n'], + ['tcsh reference', 'env.tcsh', `set ${LOWER_NAME} = $other\n`], + ['tcsh command substitution', 'env.tcsh', `set ${LOWER_NAME} = \`cat ~/.tok\`\n`], + ['sh command substitution in backticks', 'env.sh', `${NAME}=\`cat ~/.tok\`\n`], + ['ps variable', 'setup.ps1', `$env:${NAME} = $secret\n`], + ['ps sub-expression', 'setup.ps1', `$env:${NAME} = "$($x.Token)"\n`], + ['ps placeholder', 'setup.ps1', `$env:${NAME} = '${placeholder}'\n`], + ['ps Read-Host', 'setup.ps1', `$env:${NAME} = Read-Host "token"\n`], + ['ps SetEnvironmentVariable with a variable', 'setup.ps1', `[Environment]::SetEnvironmentVariable("${NAME}", $val, "User")\n`], + ['ps SetEnvironmentVariable placeholder', 'setup.ps1', `[Environment]::SetEnvironmentVariable('${NAME}', '${placeholder}')\n`], + ['cmd %reference%', 'setup.bat', `set ${NAME}=%SECRET_VAL%\n`], + ['cmd placeholder', 'setup.bat', `set ${NAME}=${placeholder}\n`], + ['cmd set /p prompt', 'setup.bat', `set /p ${NAME}=Enter token: \n`], + ['setx reference', 'setup.bat', `setx ${NAME} %TOKEN_SRC%\n`], + ['setx placeholder', 'setup.bat', `setx ${NAME} ${placeholder}\n`], + ['export of two variable names', 'env.sh', `export ${NAME} OTHER_TOKEN\n`], + ['export of one name', 'env.sh', `export ${NAME}\n`], + ['export placeholder', 'env.sh', `export ${NAME} ${placeholder}\n`], + ['Dockerfile ARG without a value', 'Dockerfile', `ARG ${NAME}\n`], + ['Dockerfile ENV reference', 'Dockerfile', `ENV ${NAME} $\{OTHER}\n`], + ['JavaScript Set', 'app.js', `const s = new Set(['${NAME}', '${value}']);\n`], + ['Python set()', 'app.py', `s = {'${NAME}', '${value}'}\nx = set(${LOWER_NAME}, ${value})\n`], + ]; + it.each(clean)('passes: %s', (_label, file, text) => { + expect(count(file, text)).toBe(0); + }); + + it('only the distinctive forms are read in source code, and set/export without flags need a shell context', () => { + expect(count('tool.py', `x = "set ${NAME} ${value}"\n`)).toBe(0); // a bare set in code is not a command + expect(count('tool.py', `x = "export ${NAME} ${value}"\n`)).toBe(0); + expect(count('tool.py', `x = "setenv ${NAME} ${value}"\n`)).toBeGreaterThan(0); // setenv is distinctive + }); + + it('fileMode: shell start-up files and the classic shells are configuration', () => { + for (const file of ['.bashrc', '.zshrc', '.profile', '.bash_profile', '.zshenv', '.cshrc', '.tcshrc', 'env.ksh', 'env.csh', 'env.tcsh', 'setup.bat', 'setup.cmd', 'fish_variables']) { + expect(fileMode(file), file).toBe('config'); + } + }); + + it('a hostile line of set/setenv/export/flags/quotes is scanned in linear time', SLOW, () => { + const started = performance.now(); + for (const [file, text] of [ + ['a.fish', `set -gx ${NAME} a `.repeat(50000)], + ['a.fish', `set ${'-a '.repeat(100000)}${NAME}`], + ['a.fish', `set${' '.repeat(200000)}x`], + ['a.csh', `setenv ${NAME} $HOME\n`.repeat(20000)], + ['a.sh', `export ${NAME} ${'a '.repeat(100000)}`], + ['a.sh', `${'set '.repeat(50000)}${NAME}`], + ['a.ps1', `Set-Item${' '.repeat(50000)}Env:${NAME}`], + ['a.ps1', `${'Set-Item '.repeat(20000)}\n`], + ['a.ps1', `[Environment]::SetEnvironmentVariable(${"'".repeat(100000)}`], + ['a.md', `${'```sh\n'.repeat(30000)}set -gx ${NAME} x\n`], + ['a.md', `${'`'.repeat(200000)}\n`], + ['fish_variables', `SETUVAR ${NAME}:x\n`.repeat(20000)], + ]) { + scanText(file, text); + } + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); +}); + +describe('review round 11 (2): values that span lines', () => { + const NAME = ['API_', 'TOKEN'].join(''); + const SECRET = ['jwt_', 'secret'].join(''); + const value = randomString(24, 8102); + const passphrase = ['correct', 'horse', 'battery', 'staple'].join(' '); + const placeholder = ['your', 'secret', 'here'].join('_'); + const Q3 = '"'.repeat(3); + const S3 = "'".repeat(3); + const count = (file, text) => scanText(file, text).length; + + const flagged = [ + // HCL / Terraform heredocs + ['tf heredoc', 'main.tf', `${SECRET} = < { + expect(count(file, text)).toBeGreaterThanOrEqual(1); + }); + + const clean = [ + ['tf placeholder', 'main.tf', `${SECRET} = < { + expect(count(file, text)).toBe(0); + }); + + it('a body inside the bound is read to its last line; one past the bound is reported instead of skipped', () => { + const filler = 'a line of ordinary filler text\n'; + // 150 lines, the secret on the last one: read in full + expect(count('main.tf', `${SECRET} = < { + expect(count('main.tf', `${SECRET} = < { + const found = scanText('main.tf', `${SECRET} = < { + for (const [file, opener] of [ + ['a.tf', `${SECRET} = < { + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + const run = (cmd, args, cwd) => spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: SLOW_TEST_MS, maxBuffer: 64 * 1024 * 1024 }); + const scan = (cwd, ...args) => run(process.execPath, [SCANNER, ...args], cwd); + const git = (cwd, ...args) => run('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', '-c', 'commit.gpgsign=false', ...args], cwd); + const write = (dir, file, content) => { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), content); + }; + const commit = (dir, message, files = {}) => { + for (const [file, content] of Object.entries(files)) write(dir, file, content); + git(dir, 'add', '-A'); + expect(git(dir, 'commit', '-q', '-m', message).status).toBe(0); + return git(dir, 'rev-parse', 'HEAD').stdout.trim(); + }; + const makeRepo = () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-bin-')); + dirs.push(dir); + expect(run('git', ['init', '-q', '-b', 'main'], dir).status).toBe(0); + return dir; + }; + const OVER = 5 * 1024 * 1024 + 4096; + // PNG-headed content with a NUL in it, N bytes of pseudo-random data (newlines every few hundred bytes, like a real image) + const png = (size, seed = 1) => { + const body = Buffer.alloc(size); + let state = seed; + for (let i = 0; i < size; i += 1) { + state = (state * 1103515245 + 12345) % 2147483648; + body[i] = (state >>> 16) & 0xff; + } + return Buffer.concat([PNG_HEAD, body]); + }; + const constantPng = (size) => Buffer.concat([PNG_HEAD, Buffer.alloc(size, 1)]); // one enormous "line": no newline byte at all + const textOver = (first = '') => `${first}${`${'x'.repeat(1023)}\n`.repeat(5 * 1024 + 8)}`; + const secretLine = `${['API_', 'KEY'].join('')}=${randomString(32, 8103)}\n`; + const base = (dir) => commit(dir, 'base', { 'base.txt': 'base\n' }); + + it.skipIf(!hasGit())('the tree scan already skips the same asset (the behaviour --range and --history now match)', SLOW, () => { + const dir = makeRepo(); + commit(dir, 'asset', { 'logo.png': png(OVER) }); + const result = scan(dir); + expect(result.status).toBe(0); + expect(result.stdout).toContain('1 skipped: 1 binary'); + }); + + it.skipIf(!hasGit())('--range: adding a binary over the size limit exits 0 and counts it as skipped: binary', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'add image', { 'assets/logo.png': png(OVER) }); + const result = scan(dir, '--range', `${from}..${head}`); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('no hits in 1 commit'); + expect(result.stdout).toContain('1 skipped: 1 binary'); + expect(result.stderr).not.toContain('INCOMPLETE'); + }); + + it.skipIf(!hasGit())('--history: adding a binary over the size limit exits 0 and counts it as skipped: binary', SLOW, () => { + const dir = makeRepo(); + base(dir); + commit(dir, 'add image', { 'assets/logo.png': png(OVER) }); + const result = scan(dir, '--history'); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('no hits in 2 commits'); + expect(result.stdout).toContain('1 skipped: 1 binary'); + }); + + it.skipIf(!hasGit())('--range and --history: modifying a binary over the size limit (both versions large)', SLOW, () => { + const dir = makeRepo(); + commit(dir, 'add image', { 'logo.png': png(OVER, 1) }); + const from = git(dir, 'rev-parse', 'HEAD').stdout.trim(); + const head = commit(dir, 'change image', { 'logo.png': png(OVER, 2) }); + const range = scan(dir, '--range', `${from}..${head}`); + expect(range.status, range.stderr).toBe(0); + expect(range.stdout).toContain('1 skipped: 1 binary'); + const history = scan(dir, '--history'); + expect(history.status, history.stderr).toBe(0); + expect(history.stdout).toContain('2 skipped: 2 binary'); + }); + + it.skipIf(!hasGit())('--range: renaming a large binary is judged by the new version, and is skipped', SLOW, () => { + const dir = makeRepo(); + const from = commit(dir, 'add image', { 'logo.png': png(OVER) }); + git(dir, 'mv', 'logo.png', 'brand.png'); + expect(git(dir, 'commit', '-q', '-m', 'rename').status).toBe(0); + const result = scan(dir, '--range', `${from}..HEAD`); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('1 skipped: 1 binary'); + }); + + it.skipIf(!hasGit())('--range: a small binary is skipped and counted, like the tree scan does', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'add icon', { 'icon.png': png(2000) }); + const result = scan(dir, '--range', `${from}..${head}`); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('1 skipped: 1 binary'); + }); + + it.skipIf(!hasGit())('--range: a large binary with no newline byte at all is skipped, and the reader keeps its memory bounded', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'add blob', { 'blob.bin': constantPng(20 * 1024 * 1024) }); + const result = scan(dir, '--range', `${from}..${head}`); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout).toContain('1 skipped: 1 binary'); + }); + + it.skipIf(!hasGit())('--range: text over the size limit still exits 2 (--range and --history)', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'add big text', { 'big.txt': textOver() }); + for (const args of [['--range', `${from}..${head}`], ['--history']]) { + const result = scan(dir, ...args); + expect(result.status, args.join(' ')).toBe(2); + expect(result.stderr).toContain('INCOMPLETE'); + expect(result.stderr).toContain('1 over the 5 MB limit'); + } + }); + + it.skipIf(!hasGit())('--range: a single added line over the size limit is unscanned too (it used to pass as clean)', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'add one huge line', { 'big.env': `${'y'.repeat(20 * 1024 * 1024)}\n` }); + const result = scan(dir, '--range', `${from}..${head}`); + expect(result.status).toBe(2); + expect(result.stderr).toContain('INCOMPLETE'); + }); + + it.skipIf(!hasGit())('--range: a lockfile over its limit still exits 2, a binary next to it does not hide it', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'add both', { 'logo.png': png(OVER), 'package-lock.json': `${'x'.repeat(1023)}\n`.repeat(16 * 1024 + 8) }); + const result = scan(dir, '--range', `${from}..${head}`); + expect(result.status).toBe(2); + expect(result.stderr).toContain('1 file version NOT scanned'); + expect(result.stderr).toContain('1 over the 16 MB lockfile limit'); + }); + + it.skipIf(!hasGit())('a NUL prefix without a known signature is NOT binary: small text with a secret is found, large text is oversize (as in the tree scan)', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'nul prefix', { 'x.env': Buffer.concat([Buffer.from([0, 0, 0, 0]), Buffer.from(secretLine)]) }); + const found = scan(dir, '--range', `${from}..${head}`); + expect(found.status).toBe(1); + expect(found.stderr).toContain('x.env secret-assignment'); + expect(scan(dir).status).toBe(1); // the tree scan agrees + const big = commit(dir, 'nul prefix, big', { 'big.dat': Buffer.concat([Buffer.from([0, 0, 0, 0]), Buffer.from(textOver())]) }); + const oversize = scan(dir, '--range', `${head}..${big}`); + expect(oversize.status).toBe(2); + expect(oversize.stderr).toContain('1 over the 5 MB limit'); + expect(scan(dir).status).toBe(1); // the tree scan reports it as oversize too + }); + + it.skipIf(!hasGit())('a PNG-headed file that holds a text secret is handled exactly as the tree scan handles it (skipped, no bypass of a NUL-only file)', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'png with text', { 'blob.dat': Buffer.concat([PNG_HEAD, Buffer.from(secretLine)]) }); + const range = scan(dir, '--range', `${from}..${head}`); + const tree = scan(dir); + expect(tree.status).toBe(0); + expect(tree.stdout).toContain('1 skipped: 1 binary'); + expect(range.status).toBe(0); + expect(range.stdout).toContain('1 skipped: 1 binary'); + }); + + it.skipIf(!hasGit())('a binary version does not hide a secret in the same commit', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'both', { 'logo.png': png(OVER), 'x.env': secretLine }); + const result = scan(dir, '--range', `${from}..${head}`); + expect(result.status).toBe(1); + expect(result.stderr).toContain('x.env secret-assignment x1'); + }); + + it.skipIf(!hasGit())('a lowered core.bigFileThreshold changes nothing: the binary is still skipped, a text version is still scanned', SLOW, () => { + const dir = makeRepo(); + git(dir, 'config', 'core.bigFileThreshold', '1k'); + const from = base(dir); + const head = commit(dir, 'add', { 'logo.png': png(100 * 1024) }); + const skipped = scan(dir, '--range', `${from}..${head}`); + expect(skipped.status, skipped.stderr).toBe(0); + expect(skipped.stdout).toContain('1 skipped: 1 binary'); + const leak = commit(dir, 'add text', { 'x.env': `${'# filler line\n'.repeat(400)}${secretLine}` }); + const found = scan(dir, '--range', `${head}..${leak}`); + expect(found.status).toBe(1); + expect(found.stderr).toContain('x.env secret-assignment x1'); + }); + + it.skipIf(!hasGit())('the report and the summary never contain the file content', SLOW, () => { + const dir = makeRepo(); + const from = base(dir); + const head = commit(dir, 'add', { 'logo.png': png(OVER), 'x.env': secretLine }); + const result = scan(dir, '--range', `${from}..${head}`); + const output = `${result.stdout}${result.stderr}`; + for (const piece of windows(secretLine.split('=')[1].trim())) expect(output).not.toContain(piece); + }); +}); + +function hasGit() { + return spawnSync('git', ['--version']).status === 0; +} + +// --------------------------------------------------------------------------- +// Review round 12: assignment operators, block scalars in split pairs, SQL password literal forms, heredoc bodies. +// Every value is assembled at runtime; the fixtures hold no secret-shaped literal. +// --------------------------------------------------------------------------- + +describe('review round 12', () => { + const NAME = secretName('JWT_', 'SECRET'); + const camel = secretName('jwt', 'Secret'); + const value = randomString(24, 12001); + const passphrase = ['correct', 'horse', 'battery', 'staple'].join(' '); + const placeholder = ['your', 'secret', 'here'].join('_'); + const count = (file, text) => scanText(file, text).length; + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + const run = (cmd, args, cwd) => spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: SLOW_TEST_MS, maxBuffer: 64 * 1024 * 1024 }); + const scan = (cwd, ...args) => run(process.execPath, [SCANNER, ...args], cwd); + const git = (cwd, ...args) => run('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', '-c', 'commit.gpgsign=false', ...args], cwd); + const makeRepo = () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-r12-')); + dirs.push(dir); + expect(run('git', ['init', '-q', '-b', 'main'], dir).status).toBe(0); + return dir; + }; + const commit = (dir, files) => { + for (const [file, content] of Object.entries(files)) { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), content); + } + git(dir, 'add', '-A'); + expect(git(dir, 'commit', '-q', '-m', 'c').status).toBe(0); + return git(dir, 'rev-parse', 'HEAD').stdout.trim(); + }; + + // ------------------------------------------------------------------------- + describe('(1) every assignment operator', () => { + const q = (v) => `'${v}'`; + const dq = (v) => `"${v}"`; + const flagged = [ + ['JS ||= on process.env', 'a.js', `process.env.${NAME} ||= ${q(value)};\n`], + ['TS ||= double quotes', 'a.ts', `process.env.${NAME} ||= ${dq(value)};\n`], + ['Ruby ||= on ENV[]', 'a.rb', `ENV['${NAME}'] ||= ${q(value)}\n`], + ['JS ??= on a config object', 'a.js', `config.${NAME} ??= ${q(value)};\n`], + ['JS ??= without blanks', 'a.js', `config.${NAME}??=${q(value)};\n`], + ['JS &&=', 'a.js', `config.${NAME} &&= ${q(value)};\n`], + ['Go :=', 'a.go', `${NAME} := ${dq(value)}\n`], + ['Python walrus', 'a.py', `if (${camel} := ${dq(value)}):\n pass\n`], + ['Make :=', 'Makefile', `${NAME} := ${value}\n`], + ['Make ::=', 'Makefile', `${NAME} ::= ${value}\n`], + ['Make ?=', 'Makefile', `${NAME} ?= ${value}\n`], + ['shell +=', 'a.sh', `${NAME}+=${value}\n`], + ['JS +=', 'a.js', `${camel} += ${dq(value)};\n`], + ['JS -=', 'a.js', `${camel} -= ${dq(value)};\n`], + ['JS *=', 'a.js', `${camel} *= ${dq(value)};\n`], + ['JS **=', 'a.js', `${camel} **= ${dq(value)};\n`], + ['JS |=', 'a.js', `${camel} |= ${dq(value)};\n`], + ['PHP .=', 'a.php', `$${camel} .= ${q(value)};\n`], + ['PHP .= without a blank', 'a.php', `$${camel}.=${q(value)};\n`], + ['Perl .=', 'a.pl', `$${camel} .= ${dq(value)};\n`], + ['R <-', 'a.R', `${camel} <- ${dq(value)}\n`], + ['R <- without blanks', 'a.R', `${camel}<-${q(value)}\n`], + ['R <<-', 'a.R', `${camel} <<- ${dq(value)}\n`], + ['PHP array =>', 'a.php', `'${NAME}' => ${q(value)},\n`], + ['Ruby hash =>', 'a.rb', `{ :${camel} => ${q(value)} }\n`], + ['Scala ->', 'a.scala', `Map(${dq(NAME)} -> ${dq(value)})\n`], + ['Kotlin to', 'a.kt', `mapOf(${dq(NAME)} to ${dq(value)})\n`], + ['plain = with blanks', 'a.js', `${camel} = ${dq(value)};\n`], + ['JSON colon', 'a.json', `{"${NAME}": ${dq(value)}}\n`], + ['Lua local', 'a.lua', `local ${camel} = ${dq(value)}\n`], + ['Lua table key', 'a.lua', `config[${dq(NAME)}] = ${dq(value)}\n`], + ['Lua attribute', 'a.lua', `local ${camel} = ${dq(value)}\n`], + ['Nim export marker', 'a.nim', `const ${camel}* = ${dq(value)}\n`], + ['Nim let', 'a.nim', `let ${camel} = ${dq(value)}\n`], + ['Kotlin val', 'a.kt', `val ${camel} = ${dq(value)}\n`], + ['Kotlin const val', 'a.kt', `const val ${NAME} = ${dq(value)}\n`], + ['Kotlin typed nullable', 'a.kt', `val ${camel}: String? = ${dq(value)}\n`], + ['Elixir keyword', 'a.ex', `config :app, ${camel}: ${dq(value)}\n`], + ['Elixir module attribute', 'a.ex', `@${camel} ${dq(value)}\n`], + ['Elixir map arrow', 'a.ex', `%{${dq(NAME)} => ${dq(value)}}\n`], + ['Scala val', 'a.scala', `val ${camel}: String = ${dq(value)}\n`], + ['Swift let', 'a.swift', `let ${camel} = ${dq(value)}\n`], + ['Swift static let typed', 'a.swift', `static let ${camel}: String = ${dq(value)}\n`], + ['Rust const &str', 'a.rs', `const ${NAME}: &str = ${dq(value)};\n`], + ['Rust static with a lifetime', 'a.rs', `static ${NAME}: &'static str = ${dq(value)};\n`], + ['Go const', 'a.go', `const ${NAME} = ${dq(value)}\n`], + ['Go const typed', 'a.go', `const ${NAME} string = ${dq(value)}\n`], + ['Go var typed', 'a.go', `var ${NAME} string = ${dq(value)}\n`], + ['Java static final', 'a.java', `static final String ${NAME} = ${dq(value)};\n`], + ['Java private static final', 'a.java', `private static final String ${NAME} = ${dq(value)};\n`], + ['C# const', 'a.cs', `const string ${NAME} = ${dq(value)};\n`], + ['C char array', 'a.c', `static const char ${NAME}[] = ${dq(value)};\n`], + ['Zig slice type', 'a.zig', `const ${camel}: []const u8 = ${dq(value)};\n`], + ['Clojure def', 'a.clj', `(def ${camel} ${dq(value)})\n`], + ['Clojure keyword map', 'a.clj', `{:${camel} ${dq(value)}}\n`], + ['Lisp setq', 'a.el', `(setq ${camel} ${dq(value)})\n`], + ]; + it.each(flagged)('reports: %s', (_label, file, text) => { + expect(rules(file, text)).toContain('secret-assignment'); + }); + + it.each(flagged.filter((_case, index) => index % 5 === 0))('reports through --range: %s', SLOW, (_label, file, text) => { + const dir = makeRepo(); + const from = commit(dir, { 'base.txt': 'base\n' }); + const head = commit(dir, { [file]: text }); + const found = scan(dir, '--range', `${from}..${head}`); + expect(found.status, found.stderr).toBe(1); + expect(`${found.stdout}${found.stderr}`).not.toContain(value); + expect(scan(dir).status).toBe(1); + }); + + const passes = [ + ['||= a reference to another variable', 'a.js', `process.env.${NAME} ||= process.env.OTHER_SECRET;\n`], + ['??= the same variable', 'a.js', `config.${NAME} ??= process.env.${NAME};\n`], + ['Ruby ||= ENV', 'a.rb', `ENV['${NAME}'] ||= ENV['OTHER']\n`], + ['??= a placeholder', 'a.js', `config.${NAME} ??= ${q(placeholder)};\n`], + ['??= an angle-bracket marker', 'a.js', `config.${NAME} ??= '';\n`], + ['??= a phrase in code (text, as with =)', 'a.js', `config.${NAME} ??= ${dq(passphrase)};\n`], + ['Go := reading the environment', 'a.go', `${NAME} := os.Getenv(${dq(NAME)})\n`], + ['R <- reading the environment', 'a.R', `${camel} <- Sys.getenv(${dq(NAME)})\n`], + ['Rust const from env!', 'a.rs', `const ${NAME}: &str = env!(${dq(NAME)});\n`], + ['+= a number', 'a.js', `${camel} += 1;\n`], + ['a comparison is not an assignment', 'a.js', `if (${camel} != ${dq(value)}) {}\n`], + ['-> without a quote after it', 'a.php', `$this->${camel} = $other;\n`], + ['a Go channel send of a name that is not secret-like', 'a.go', `events <- ${dq(value)}\n`], + ['to as an ordinary word in prose', 'a.md', `send the ${camel} to "${value}"\n`], + ['=> in a lambda over an identifier', 'a.js', `const f = (${camel}) => ${camel}.length;\n`], + ]; + it.each(passes)('passes: %s', (_label, file, text) => { + expect(count(file, text)).toBe(0); + }); + + it('a type word or annotation in front of the operator does not hide a later assignment', () => { + expect(count('a.rs', `const A: &str = "x";\nconst ${NAME}: &str = ${dq(value)};\n`)).toBe(1); + expect(count('a.go', `var A string\nvar ${NAME} string = ${dq(value)}\n`)).toBe(1); + }); + + it('the new forms stay linear on hostile lines', SLOW, () => { + const hostile = [ + ['a.ts', `${camel}: ${"&'a mut str ".repeat(30000)}\n`], + ['a.go', `${`${camel} string `.repeat(30000)}\n`], + ['a.js', `${`${camel} ||= ??= &&= ::= `.repeat(20000)}\n`], + ['a.kt', `${`${dq(camel)} to `.repeat(30000)}\n`], + ['a.clj', `${`(def ^:a ^:b ${camel} `.repeat(20000)}\n`], + ['a.ex', `${`@${camel} `.repeat(30000)}\n`], + ['a.nim', `${`${camel}* ${camel}? ${camel}[] ${camel} `.repeat(15000)}\n`], + ['a.R', `${`${camel} <<- `.repeat(30000)}\n`], + ['a.js', `${camel}${' '.repeat(300000)}x\n`], + ]; + const started = performance.now(); + for (const [file, text] of hostile) scanText(file, text); + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + // ------------------------------------------------------------------------- + describe('(2) block scalars and multi-line values in split name/value pairs', () => { + const markers = ['|', '>', '|-', '>-', '|+', '>+', '|2', '|2-', '>-2']; + const bodies = [ + ['a passphrase', passphrase], + ['a random token', value], + ]; + const cases = markers.flatMap((marker) => + bodies.flatMap(([kind, body]) => [ + [`Kubernetes name then value ${marker}, ${kind}`, 'd.yaml', `env:\n - name: ${NAME}\n value: ${marker}\n ${body}\n`], + [`value ${marker} before the name, ${kind}`, 'd.yaml', `env:\n - value: ${marker}\n ${body}\n name: ${NAME}\n`], + ]), + ); + it.each(cases)('reports: %s', (_label, file, text) => { + expect(rules(file, text)).toContain('secret-name-value-pair'); + }); + + const siblings = [ + ['secret on the second body line', 'd.yaml', `- name: ${NAME}\n value: |\n a line of ordinary text\n ${value}\n`], + ['passphrase on the second body line', 'd.yaml', `- name: ${NAME}\n value: |\n first words\n ${passphrase}\n`], + ['CRLF line ends', 'd.yaml', `- name: ${NAME}\r\n value: |-\r\n ${passphrase}\r\n`], + ['a YAML anchor before the header', 'd.yaml', `- name: ${NAME}\n value: &a |\n ${value}\n`], + ['a comment after the header', 'd.yaml', `- name: ${NAME}\n value: | # note\n ${passphrase}\n`], + ['Helm values: mapping key with a nested value block', 'values.yaml', `${NAME}:\n value: |\n ${passphrase}\n`], + ['Helm values: the same one level down', 'values.yaml', `secrets:\n ${NAME}:\n value: |-\n ${value}\n`], + ['mapping key with a !!str value block', 'a.yaml', `${NAME}:\n value: !!str |\n ${value}\n`], + ['GitHub Actions with: name and value', '.github/workflows/ci.yml', ` with:\n name: ${NAME}\n value: |\n ${value}\n`], + ['GitHub Actions with: value before name', '.github/workflows/ci.yml', ` with:\n value: >-\n ${passphrase}\n name: ${NAME}\n`], + ['CloudFormation Value: !Sub |', 'stack.yaml', `Variables:\n - Name: ${NAME}\n Value: !Sub |\n ${value}\n`], + ['CloudFormation Value: !Sub > passphrase', 'stack.yaml', `- Name: ${NAME}\n Value: !Sub >-\n ${passphrase}\n`], + ['CloudFormation Value first', 'stack.yaml', `- Value: !Sub |\n ${value}\n Name: ${NAME}\n`], + ['CloudFormation ParameterKey / ParameterValue', 'stack.yaml', `- ParameterKey: ${NAME}\n ParameterValue: |\n ${value}\n`], + ['docker-compose environment list of name/value maps', 'docker-compose.yml', `services:\n a:\n environment:\n - name: ${NAME}\n value: |\n ${passphrase}\n`], + ['docker-compose environment map', 'docker-compose.yml', `services:\n a:\n environment:\n ${NAME}: |\n ${passphrase}\n`], + ['GitHub Actions env: map', '.github/workflows/ci.yml', ` env:\n ${NAME}: >-\n ${value}\n`], + ['a quote closed on a later line', 'a.yaml', `- name: ${NAME}\n value: "first line\n ${passphrase}"\n`], + ['a plain scalar folded over lines', 'a.yaml', `- name: ${NAME}\n value: first\n ${passphrase}\n`], + ['Terraform heredoc value after the name', 'a.tf', `variable {\n name = "${NAME}"\n value = <${NAME}\n first\n ${passphrase}\n\n`], + ['value-first with a body of 30 lines', 'd.yaml', `env:\n - value: |\n${' a filler line of text\n'.repeat(30)} ${passphrase}\n name: ${NAME}\n`], + ['value-first with a blank line in the body', 'd.yaml', `env:\n - value: |\n first\n\n ${passphrase}\n name: ${NAME}\n`], + ['name-first with a body of 150 lines, the secret last', 'd.yaml', `env:\n - name: ${NAME}\n value: |\n${' a filler line of text\n'.repeat(150)} ${value}\n`], + ]; + it.each(siblings)('reports: %s', (_label, file, text) => { + expect(count(file, text)).toBeGreaterThanOrEqual(1); + }); + + const clean = [ + ['a placeholder body', 'd.yaml', `- name: ${NAME}\n value: |\n ${placeholder}\n`], + ['an empty block followed by a dedented key', 'd.yaml', `- name: ${NAME}\n value: |\nnext: ${value}\n`], + ['documentation about the credential', 'd.yaml', `- name: ${NAME}\n value: >\n Set this to the signing secret from the dashboard.\n`], + ['a body that ends before a secret in a sibling key', 'd.yaml', `- name: ${NAME}\n value: |\n ${placeholder}\n other: ${value}\n`], + ['a name that is not secret-like', 'd.yaml', `- name: FEATURE_FLAGS\n value: |\n ${passphrase}\n`], + ['a path in a *_FILE variable', 'd.yaml', `- name: ${NAME}_FILE\n value: |\n /var/run/secrets/jwt/key\n`], + ['valueFrom instead of a value', 'd.yaml', `- name: ${NAME}\n valueFrom:\n secretKeyRef:\n name: app\n key: jwt\n`], + ['a template reference', 'values.yaml', `${NAME}:\n value: |\n {{ .Values.jwt }}\n`], + ['a CloudFormation reference', 'stack.yaml', `- Name: ${NAME}\n Value: !Sub |\n \${SecretParam}\n`], + ['prose lines in an escaped JSON string', 'a.json', `[{"name":"${NAME}","value":"Set the value of this secret\\nfrom the dashboard"}]\n`], + ['a later, unrelated pair', 'd.yaml', `- name: ${NAME}\n valueFrom: x\n- name: NOTE\n value: |\n ${passphrase}\n`], + ]; + it.each(clean)('passes: %s', (_label, file, text) => { + expect(count(file, text)).toBe(0); + }); + + it('an allow marker on the header line, on a body line or on the name line silences the pair', () => { + const M = ALLOW_MARKER; + expect(count('d.yaml', `- name: ${NAME}\n value: | # ${M}\n ${passphrase}\n`)).toBe(0); + expect(count('d.yaml', `- name: ${NAME}\n value: |\n ${passphrase} # ${M}\n`)).toBe(0); + expect(count('d.yaml', `- name: ${NAME}\n value: |\n ${passphrase}\n # ${M}\n`)).toBe(0); + expect(count('d.yaml', `- name: ${NAME} # ${M}\n value: |\n ${passphrase}\n`)).toBe(0); + // the fixtures are real: without the marker they are findings + expect(count('d.yaml', `- name: ${NAME}\n value: |\n ${passphrase} # note\n`)).toBe(1); + }); + + it('a marker on an unrelated line does not silence it', () => { + expect(count('d.yaml', `# ${ALLOW_MARKER}\n- name: ${NAME}\n type: plain\n value: |\n ${passphrase}\n`)).toBe(1); + }); + + it('a block that does not end within the bounds is reported (fail closed), not skipped', () => { + const filler = ' a filler line of ordinary text\n'; + expect(count('d.yaml', `env:\n - name: ${NAME}\n value: |\n${filler.repeat(300)}`)).toBeGreaterThanOrEqual(1); + // a long body under a name that is not secret-like is not reported + expect(count('d.yaml', `env:\n - name: FEATURE_FLAGS\n value: |\n${filler.repeat(300)}`)).toBe(0); + }); + + it('the report names the pair rule and the name line, never the value', () => { + const found = scanText('d.yaml', `env:\n - name: ${NAME}\n value: |\n ${value}\n`); + expect(found).toEqual([{ path: 'd.yaml', line: 2, rule: 'secret-name-value-pair' }]); + expect(JSON.stringify(found)).not.toContain(value.slice(0, 8)); + }); + + const order = ['name-first', 'value-first']; + const doc = (which, { name, marker, body }) => + which === 'name-first' + ? `env:\n - name: ${name}\n value: ${marker}\n ${body}\n` + : `env:\n - value: ${marker}\n ${body}\n name: ${name}\n`; + const changes = [ + ['the commit adds the body', { name: NAME, marker: '>-', body: placeholder }, { name: NAME, marker: '>-', body: passphrase }], + ['the commit adds the name', { name: 'FEATURE_NAME', marker: '>-', body: passphrase }, { name: NAME, marker: '>-', body: passphrase }], + ['the commit adds the marker', { name: NAME, marker: 'plain', body: passphrase }, { name: NAME, marker: '>-', body: passphrase }], + ]; + const fixed = { name: 'FEATURE_NAME', marker: '|', body: 'unrelated' }; + const historyCases = order.flatMap((which) => changes.flatMap(([label, before, after]) => ['--history', '--range'].map((mode) => [`${which}: ${label} (${mode})`, which, before, after, mode]))); + it.each(historyCases)('blames the right commit: %s', SLOW, (_label, which, before, after, mode) => { + const dir = makeRepo(); + const first = commit(dir, { 'd.yaml': doc(which, before) }); + const leak = commit(dir, { 'd.yaml': doc(which, after) }); + const tip = commit(dir, { 'd.yaml': doc(which, fixed) }); + const args = mode === '--range' ? ['--range', `${first}..${tip}`] : ['--history']; + const found = scan(dir, ...args); + expect(found.status, found.stderr).toBe(1); + expect(`${found.stdout}${found.stderr}`).not.toContain(passphrase); + // a range that starts after the leak, and the clean tip itself, report nothing + expect(scan(dir, '--range', `${leak}..${tip}`).status).toBe(0); + expect(scan(dir).status).toBe(0); + }); + + it('the tree scan and --range agree on a block scalar pair', SLOW, () => { + const dir = makeRepo(); + const from = commit(dir, { 'base.txt': 'base\n' }); + const head = commit(dir, { 'k8s/deploy.yaml': doc('name-first', { name: NAME, marker: '|', body: passphrase }) }); + expect(scan(dir).status).toBe(1); + const found = scan(dir, '--range', `${from}..${head}`); + expect(found.status).toBe(1); + expect(found.stderr).toContain('k8s/deploy.yaml secret-name-value-pair'); + }); + + it('dense secret-like names with block scalars are read within a budget', SLOW, () => { + const started = performance.now(); + for (const text of [ + `- name: ${NAME}\n value: |\n`.repeat(20000), + `- value: |\n x\n name: ${NAME}\n`.repeat(20000), + ` ${NAME}:\n value: |\n a b\n`.repeat(20000), + `- name: ${NAME}\n value: "a\n`.repeat(20000), + ` filler\n`.repeat(50000) + ` name: ${NAME}\n`, + `- name: ${NAME}\n value: |\n${' some words here\n'.repeat(40000)}`, + ]) { + scanText('d.yaml', text); + } + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + // ------------------------------------------------------------------------- + describe('(3) SQL password literals in every quoted and unquoted form', () => { + const H = ['*', randomString(40, 12002, HEX.toUpperCase())].join(''); + const newer = randomString(12, 12003); + const shortWord = ['hunter', '2hunter2'].join(''); + const ref = (name) => ['$', '{', name, '}'].join(''); + const D = '$'; + const flagged = [ + ['Oracle, double quotes', 'a.sql', `CREATE USER app IDENTIFIED BY "${value}";\n`], + ['Oracle, double quotes in a .txt', 'a.txt', `CREATE USER app IDENTIFIED BY "${value}";\n`], + ['Oracle, double quotes in Markdown', 'a.md', `CREATE USER app IDENTIFIED BY "${value}";\n`], + ['Oracle, double quotes inside Python', 'a.py', `cur.execute('create user app identified by "${value}"')\n`], + ['Oracle, unquoted', 'a.sql', `CREATE USER app IDENTIFIED BY ${value};\n`], + ['Oracle, unquoted in Markdown', 'a.md', `ALTER USER app IDENTIFIED BY ${value};\n`], + ['Oracle, GRANT with an unquoted password', 'a.md', `GRANT CONNECT TO app IDENTIFIED BY ${value};\n`], + ['Oracle, REPLACE clause (old password)', 'a.sql', `ALTER USER app IDENTIFIED BY '${newer}' REPLACE '${value}';\n`], + ['Oracle, unquoted REPLACE', 'a.sql', `ALTER USER app IDENTIFIED BY ${newer} REPLACE ${value};\n`], + ['Oracle, IDENTIFIED BY VALUES hash', 'a.sql', `CREATE USER app IDENTIFIED BY VALUES 'S:${value}';\n`], + ['MySQL, single quotes', 'a.sql', `CREATE USER 'app'@'%' IDENTIFIED BY '${value}';\n`], + ['MySQL, double quotes', 'a.sql', `CREATE USER 'app'@'%' IDENTIFIED BY "${value}";\n`], + ['MySQL, backticks', 'a.sql', `CREATE USER 'app'@'%' IDENTIFIED BY \`${value}\`;\n`], + ['MySQL, IDENTIFIED WITH plugin BY', 'a.sql', `CREATE USER 'app'@'%' IDENTIFIED WITH caching_sha2_password BY '${value}';\n`], + ['MySQL, IDENTIFIED WITH plugin AS hash', 'a.sql', `CREATE USER 'app'@'%' IDENTIFIED WITH mysql_native_password AS '${H}';\n`], + ['MySQL, IDENTIFIED BY PASSWORD hash', 'a.sql', `GRANT ALL ON *.* TO 'app'@'%' IDENTIFIED BY PASSWORD '${H}';\n`], + ['MySQL, GRANT ... IDENTIFIED BY', 'a.sql', `GRANT ALL ON db.* TO 'app'@'%' IDENTIFIED BY '${value}';\n`], + ['MariaDB, IDENTIFIED VIA ... USING PASSWORD()', 'a.sql', `CREATE USER app IDENTIFIED VIA mysql_native_password USING PASSWORD('${value}');\n`], + ['MySQL, SET PASSWORD FOR ... = PASSWORD()', 'a.sql', `SET PASSWORD FOR 'app'@'%' = PASSWORD('${value}');\n`], + ['MySQL, SET PASSWORD FOR ... = PASSWORD() inside Java', 'A.java', `stmt.execute("SET PASSWORD FOR 'app'@'%' = PASSWORD('${value}')");\n`], + ['MySQL, OLD_PASSWORD()', 'a.sql', `SET PASSWORD FOR 'app'@'%' = OLD_PASSWORD('${value}');\n`], + ['MySQL, SET PASSWORD FOR ... = literal', 'a.sql', `SET PASSWORD FOR 'app'@'%' = '${value}';\n`], + ['MySQL, SET PASSWORD = literal inside Python', 'a.py', `cur.execute("SET PASSWORD = '${value}'")\n`], + ['MySQL, ALTER USER ... IDENTIFIED BY', 'a.sql', `ALTER USER 'app'@'localhost' IDENTIFIED BY '${value}';\n`], + ['MySQL, UPDATE mysql.user SET ... = PASSWORD()', 'a.sql', `UPDATE mysql.user SET authentication_string=PASSWORD('${value}') WHERE User='root';\n`], + ['MySQL, UPDATE mysql.user inside PHP', 'a.php', `$db->query("UPDATE mysql.user SET Password=PASSWORD('${value}') WHERE User='root'");\n`], + ['PostgreSQL, single quotes', 'a.sql', `ALTER ROLE app WITH PASSWORD '${value}';\n`], + ['PostgreSQL, double quotes', 'a.sql', `ALTER ROLE app WITH PASSWORD "${value}";\n`], + ['PostgreSQL, $$dollar$$ quoting', 'a.sql', `ALTER ROLE app WITH PASSWORD ${D}${D}${value}${D}${D};\n`], + ['PostgreSQL, $tag$dollar$tag$ quoting', 'a.sql', `ALTER ROLE app WITH PASSWORD ${D}pw${D}${value}${D}pw${D};\n`], + ['PostgreSQL, E string', 'a.sql', `ALTER ROLE app WITH PASSWORD E'${value}';\n`], + ['PostgreSQL, N string', 'a.sql', `ALTER ROLE app WITH PASSWORD N'${value}';\n`], + ['PostgreSQL, U& string', 'a.sql', `ALTER ROLE app WITH PASSWORD U&'${value}';\n`], + ['PostgreSQL, ENCRYPTED PASSWORD', 'a.sql', `CREATE ROLE app WITH LOGIN ENCRYPTED PASSWORD '${value}';\n`], + ['PostgreSQL, ENCRYPTED PASSWORD without WITH', 'a.sql', `ALTER ROLE app ENCRYPTED PASSWORD '${value}';\n`], + ['PostgreSQL, UNENCRYPTED PASSWORD', 'a.sql', `CREATE ROLE app UNENCRYPTED PASSWORD '${value}';\n`], + ['PostgreSQL, LOGIN PASSWORD in Markdown', 'a.md', `CREATE ROLE app LOGIN PASSWORD '${value}';\n`], + ['PostgreSQL, CREATE USER with options and double quotes', 'a.md', `CREATE USER app SUPERUSER CREATEDB PASSWORD "${value}";\n`], + ['ALTER USER ... SET PASSWORD (Snowflake)', 'a.sql', `ALTER USER app SET PASSWORD = '${value}';\n`], + ['ALTER USER ... SET PASSWORD in Markdown', 'a.md', `ALTER USER app SET PASSWORD = '${value}';\n`], + ['ALTER USER ... SET PASSWORD with backticks', 'a.md', `ALTER USER app SET PASSWORD = \`${value}\`;\n`], + ['CREATE USER ... PASSWORD = (Snowflake)', 'a.md', `CREATE USER app PASSWORD = '${value}' MUST_CHANGE_PASSWORD = TRUE;\n`], + ['SQL Server, WITH PASSWORD =', 'a.sql', `CREATE LOGIN app WITH PASSWORD = '${value}';\n`], + ['SQL Server, N string', 'a.sql', `CREATE LOGIN app WITH PASSWORD = N'${value}';\n`], + ['SQL Server, N string in Markdown', 'a.md', `CREATE LOGIN app WITH PASSWORD = N'${value}' MUST_CHANGE;\n`], + ['SQL Server, CREATE USER WITH PASSWORD', 'a.md', `CREATE USER app WITH PASSWORD = '${value}';\n`], + ['SQL Server, OLD_PASSWORD', 'a.md', `ALTER LOGIN app WITH PASSWORD = '${newer}' OLD_PASSWORD = '${value}';\n`], + ['SQL Server, LOGIN ... PASSWORD with double quotes', 'a.md', `CREATE LOGIN app LOGIN PASSWORD = "${value}";\n`], + ['MongoDB, createUser with double quotes', 'a.js', `db.createUser({user: "app", pwd: "${value}", roles: ["readWrite"]});\n`], + ['MongoDB, createUser with single quotes', 'a.js', `db.createUser({user: 'app', pwd: '${value}', roles: []});\n`], + ['MongoDB, createUser with quoted keys', 'a.js', `db.createUser({"user": "app", "pwd": "${value}"});\n`], + ['MongoDB, a passphrase over lines', 'init.js', `db.getSiblingDB("admin").createUser({\n user: "root",\n pwd: "${passphrase}",\n roles: ["root"]\n});\n`], + ['MongoDB, a short non-random password', 'init.js', `db.createUser({user: "u", pwd: "${shortWord}"});\n`], + ['MongoDB, mongosh --eval', 'init.sh', `mongosh --eval 'db.createUser({user:"app",pwd:"${value}",roles:[]})'\n`], + ['MongoDB, updateUser', 'a.js', `db.updateUser("app", {pwd: "${value}"});\n`], + ['MongoDB, changeUserPassword', 'a.js', `db.changeUserPassword("app", "${value}");\n`], + ]; + it.each(flagged)('reports: %s', (_label, file, text) => { + expect(rules(file, text)).toContain('sql-password-literal'); + }); + + it.each(flagged.filter((_case, index) => index % 6 === 0))('reports through --range: %s', SLOW, (_label, file, text) => { + const dir = makeRepo(); + const from = commit(dir, { 'base.txt': 'base\n' }); + const head = commit(dir, { [file]: text }); + const found = scan(dir, '--range', `${from}..${head}`); + expect(found.status, found.stderr).toBe(1); + expect(found.stderr).toContain('sql-password-literal'); + expect(`${found.stdout}${found.stderr}`).not.toContain(value); + }); + + const clean = [ + ['Oracle placeholder in angle brackets', 'a.sql', `CREATE USER app IDENTIFIED BY '';\n`], + ['Oracle placeholder in double quotes', 'a.sql', `CREATE USER app IDENTIFIED BY "";\n`], + ['Oracle ${...} reference', 'a.sql', `CREATE USER app IDENTIFIED BY "${ref('DB_PASSWORD')}";\n`], + ['Oracle bind :name', 'a.sql', 'CREATE USER app IDENTIFIED BY :pw;\n'], + ['quoted :name', 'a.sql', `ALTER ROLE app WITH PASSWORD ':pw';\n`], + ['question mark', 'a.sql', `ALTER ROLE app WITH PASSWORD '?';\n`], + ['positional $1, quoted', 'a.sql', `ALTER ROLE app WITH PASSWORD '${D}1';\n`], + ['positional $1, bare', 'a.sql', `ALTER ROLE app WITH PASSWORD ${D}1;\n`], + ['%s format marker', 'a.py', `cur.execute("ALTER ROLE app WITH PASSWORD '%s'")\n`], + ['Oracle ?', 'a.sql', 'CREATE USER app IDENTIFIED BY ?;\n'], + ['SQL*Plus &var', 'a.sql', 'CREATE USER app IDENTIFIED BY &pw;\n'], + ['SQL*Plus &&var', 'a.sql', 'CREATE USER app IDENTIFIED BY &&pw;\n'], + ['changeme in double quotes', 'a.sql', `CREATE USER app IDENTIFIED BY "changeme";\n`], + ['your_password unquoted', 'a.sql', 'CREATE USER app IDENTIFIED BY your_password;\n'], + ['a placeholder in $$ quoting', 'a.sql', `ALTER ROLE app WITH PASSWORD ${D}${D}${D}${D};\n`], + ['a placeholder in backticks', 'a.sql', "CREATE USER 'a'@'%' IDENTIFIED BY `changeme`;\n"], + ['MongoDB placeholder', 'a.js', 'db.createUser({user: "app", pwd: "", roles: []});\n'], + ['MongoDB env reference', 'a.js', 'db.createUser({user: "app", pwd: process.env.MONGO_PWD, roles: []});\n'], + ['MongoDB ${...} reference', 'a.js', `db.createUser({user: "app", pwd: "${ref('MONGO_PWD')}", roles: []});\n`], + ['PASSWORD() with a placeholder', 'a.sql', `SET PASSWORD FOR 'a'@'h' = PASSWORD('');\n`], + ['prose: identified by', 'a.md', 'Users are identified by their email address.\n'], + ['prose: identified by a word', 'a.md', `Each user is identified by ${value} in the logs.\n`], + ['prose: a bare password in Markdown', 'a.md', `the password '${value}' was shown\n`], + ['password_encryption setting', 'a.sql', "SET password_encryption = 'scram-sha-256';\n"], + ['IDENTIFIED BY VALUES with nothing after', 'a.sql', 'CREATE USER app IDENTIFIED BY VALUES ;\n'], + ['IDENTIFIED EXTERNALLY', 'a.sql', 'CREATE USER app IDENTIFIED EXTERNALLY;\n'], + ['IDENTIFIED BY RANDOM PASSWORD', 'a.sql', 'CREATE USER app IDENTIFIED BY RANDOM PASSWORD;\n'], + ['IDENTIFIED WITH a plugin only', 'a.sql', 'CREATE USER app IDENTIFIED WITH auth_socket;\n'], + ['Markdown code spans around the word password', 'a.md', 'Use `WITH PASSWORD` and then `the value` in the statement.\n'], + ['a password() helper in JavaScript', 'a.js', `form.password('${value}');\n`], + ['set password in prose', 'a.md', "Then set password to something else, for example 'x1'\n"], + ]; + it.each(clean)('passes: %s', (_label, file, text) => { + expect(rules(file, text)).not.toContain('sql-password-literal'); + }); + + it('the report names the rule and the line, never the password', () => { + const found = scanText('a.sql', `-- users\nCREATE USER app IDENTIFIED BY "${value}";\n`); + expect(found).toEqual([{ path: 'a.sql', line: 2, rule: 'sql-password-literal' }]); + }); + + it('an allow marker silences a statement', () => { + expect(count('a.sql', `CREATE USER app IDENTIFIED BY "${value}"; -- ${ALLOW_MARKER}\n`)).toBe(0); + }); + + it('the new forms stay linear on hostile input', SLOW, () => { + const rep = (s, n) => s.repeat(n); + const hostile = [ + rep('create user ', 30000), + rep('identified by ', 30000), + rep('identified with a by ', 20000), + rep('password ', 30000), + rep('set password for ', 20000), + `set password for ${rep("'a' ", 60000)}`, + rep(`password ${D}a${D}`, 20000), + rep(`password ${D}${D}${'x'.repeat(4000)}\n`, 300), + `password '${"''".repeat(150000)}`, + `identified by "${'a'.repeat(300000)}`, + `identified by \`${'a'.repeat(300000)}`, + `identified by ${rep('replace ', 30000)}`, + rep('pwd: ', 60000), + `createUser ${rep('pwd: ', 60000)}`, + rep('db.auth(', 40000), + rep('password(', 40000), + ]; + const started = performance.now(); + for (const file of ['a.sql', 'a.md', 'a.js']) for (const text of hostile) scanText(file, text); + expect(performance.now() - started).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + // ------------------------------------------------------------------------- + describe('(4) heredoc bodies with whitespace in secret CLI commands', () => { + const tools = [ + ['vercel env add', `vercel env add ${NAME} production`], + ['gh secret set', `gh secret set ${NAME}`], + ['wrangler secret put', `wrangler secret put ${NAME}`], + ['fly secrets set', `fly secrets set ${NAME}`], + ['docker secret create', `docker secret create ${NAME} -`], + ['firebase functions:secrets:set', `firebase functions:secrets:set ${NAME}`], + ['netlify env:set', `netlify env:set ${NAME}`], + ['doppler secrets set', `doppler secrets set ${NAME}`], + ['heroku config:set', `heroku config:set ${NAME}`], + ['railway variables set', `railway variables set ${NAME}`], + ]; + const tags = [ + ['plain tag', '< + tags.map(([tag, opener, closer, indent]) => [`${tool}, ${tag}`, `${command} ${opener}\n${indent}${passphrase}\n${closer}\n`]), + ); + it.each(direct)('reports a passphrase body: %s', (_label, text) => { + expect(rules('deploy.sh', text)).toContain('secret-cli-command'); + }); + + const piped = tools.flatMap(([tool, command]) => + tags.map(([tag, opener, closer, indent]) => [`cat ${opener} | ${tool}, ${tag}`, `cat ${opener} | ${command}\n${indent}${passphrase}\n${closer}\n`]), + ); + it.each(piped)('reports a passphrase body piped in: %s', (_label, text) => { + expect(rules('deploy.sh', text)).toContain('secret-cli-command'); + }); + + const hereStrings = tools.flatMap(([tool, command]) => [ + [`${tool}, double-quoted here-string`, `${command} <<< "${passphrase}"\n`], + [`${tool}, single-quoted here-string`, `${command} <<< '${passphrase}'\n`], + [`${tool}, ANSI-C here-string`, `${command} <<< $'${passphrase}'\n`], + ]); + it.each(hereStrings)('reports a here-string: %s', (_label, text) => { + expect(rules('deploy.sh', text)).toContain('secret-cli-command'); + }); + + const bodies = [ + ['a multi-line body, the passphrase on the last line', `vercel env add ${NAME} </dev/null\nnoise\nA\nvercel env add ${NAME} < { + expect(rules('deploy.sh', text)).toContain('secret-cli-command'); + }); + + it('reports in a CI step, a Markdown fence and through --range', SLOW, () => { + const inner = `vercel env add ${NAME} < { + expect(count('deploy.sh', text)).toBe(0); + }); + + it('a body without a terminator, or with one beyond the bounds, is reported (fail closed)', () => { + expect(count('deploy.sh', `vercel env add ${NAME} < { + expect(count('deploy.sh', `vercel env add ${NAME} < { + const started = performance.now(); + scanText('deploy.sh', `vercel env add ${NAME} < { + const AUTH = ['Author', 'ization'].join(''); + const token = randomString(28, 13001); + const password = randomString(14, 13002); + const basic = Buffer.from(`alice:${password}`).toString('base64'); + const placeholderBasic = Buffer.from('user:pass').toString('base64'); + const count = (file, text) => scanText(file, text).length; + const rules = (file, text) => scanText(file, text).map((f) => f.rule); + + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + const run = (cmd, args, cwd) => spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: SLOW_TEST_MS, maxBuffer: 64 * 1024 * 1024 }); + const scan = (cwd, ...args) => run(process.execPath, [SCANNER, ...args], cwd); + const git = (cwd, ...args) => run('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', '-c', 'commit.gpgsign=false', ...args], cwd); + const makeRepo = () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-r13-')); + dirs.push(dir); + expect(run('git', ['init', '-q', '-b', 'main'], dir).status).toBe(0); + return dir; + }; + const commit = (dir, files) => { + for (const [file, content] of Object.entries(files)) { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), content); + } + git(dir, 'add', '-A'); + expect(git(dir, 'commit', '-q', '-m', 'c').status).toBe(0); + return git(dir, 'rev-parse', 'HEAD').stdout.trim(); + }; + // Tree scan, --range and --history each report the change, and none of them prints the secret. + const expectReported = (rule, file, clean, leaked, secret) => { + const dir = makeRepo(); + const from = commit(dir, { [file]: clean }); + const head = commit(dir, { [file]: leaked }); + for (const args of [[], ['--range', `${from}..${head}`], ['--history']]) { + const found = scan(dir, ...args); + expect(found.status, `${args.join(' ')}: ${found.stderr}`).toBe(1); + expect(found.stderr).toContain(file); + expect(found.stderr).toContain(rule); + expect(`${found.stdout}${found.stderr}`).not.toContain(secret); + } + // the same file with the clean text is not reported + const empty = scan(dir, '--range', `${head}..${head}`); + expect(empty.status, empty.stderr).toBe(0); // clean, not an error exit (2) + }; + + // ------------------------------------------------------------------------- + describe('(1) Authorization headers and other HTTP credentials', () => { + const literal = [ + ['header text, Bearer', 'notes.txt', `${AUTH}: Bearer ${token}\n`], + ['header text, Basic', 'notes.txt', `${AUTH}: Basic ${basic}\n`], + ['header text, token scheme', 'notes.txt', `${AUTH}: token ${token}\n`], + ['header text, ApiKey scheme', 'notes.txt', `${AUTH}: ApiKey ${token}\n`], + ['header text, Api-Key scheme', 'notes.txt', `${AUTH}: Api-Key ${token}\n`], + ['header text, Negotiate', 'notes.txt', `${AUTH}: Negotiate ${token}${token}\n`], + ['header text, no scheme', 'notes.txt', `${AUTH}: ${token}\n`], + ['lower-case header name', 'notes.txt', `${AUTH.toLowerCase()}: bearer ${token}\n`], + ['Proxy-Authorization', 'notes.txt', `Proxy-${AUTH}: Basic ${basic}\n`], + ['Markdown fence', 'README.md', `\`\`\`\n${AUTH}: Bearer ${token}\n\`\`\`\n`], + ['curl -H, double quotes', 'run.sh', `curl -H "${AUTH}: Bearer ${token}" https://api.internal.corp/v1\n`], + ['curl -H, single quotes', 'run.sh', `curl -H '${AUTH}: Bearer ${token}' https://api.internal.corp/v1\n`], + ['curl --header, Basic', 'run.sh', `curl --header "${AUTH}: Basic ${basic}" https://api.internal.corp/v1\n`], + ['JS object', 'src/api.js', `fetch(url, { headers: { ${AUTH}: 'Bearer ${token}' } });\n`], + ['JS object, quoted name', 'src/api.js', `fetch(url, { headers: { '${AUTH}': "Bearer ${token}" } });\n`], + ['JS headers.set', 'src/api.js', `res.headers.set('${AUTH}', 'Bearer ${token}');\n`], + ['JS setHeader, Token scheme', 'src/api.ts', `req.setHeader("${AUTH}", "Token ${token}");\n`], + ['JS headers.append, template literal', 'src/api.ts', `headers.append('${AUTH}', \`Bearer ${token}\`);\n`], + ['Python dict', 'client.py', `requests.get(url, headers={"${AUTH}": "Bearer ${token}"})\n`], + ['Python subscript', 'client.py', `session.headers["${AUTH}"] = "Basic ${basic}"\n`], + ['Ruby hash rocket', 'client.rb', `headers = { '${AUTH}' => "Bearer ${token}" }\n`], + ['JSON', 'requests.json', `{"${AUTH}": "Bearer ${token}"}\n`], + ['JSON, nested Basic', 'requests.json', `{"headers": {"${AUTH}": "Basic ${basic}"}}\n`], + ['YAML', 'ci.yml', `headers:\n ${AUTH}: Bearer ${token}\n`], + ['YAML, quoted', 'ci.yml', ` ${AUTH}: "Bearer ${token}"\n`], + ['YAML list item', 'ci.yml', `headers:\n - "${AUTH}: Bearer ${token}"\n`], + ['.http file', 'api.http', `GET https://api.internal.corp/v1\n${AUTH}: Bearer ${token}\n`], + ['git extraheader', 'setup.sh', `git config http.extraheader "${AUTH}: Basic ${basic}"\n`], + ['nginx proxy_set_header', 'nginx.conf', `proxy_set_header ${AUTH} "Bearer ${token}";\n`], + ['Apache RequestHeader', '.htaccess', `RequestHeader set ${AUTH} "Bearer ${token}"\n`], + ['HAProxy set-header', 'haproxy.cfg', `backend b\n http-request set-header ${AUTH} "Bearer ${token}"\n`], + ['Digest response', 'notes.txt', `${AUTH}: Digest username="alice", realm="api", nonce="${randomString(16, 13003)}", response="${randomString(32, 13004, HEX)}"\n`], + ['Python auth tuple', 'client.py', `requests.get(url, auth=("alice", "${password}"))\n`], + ['Python HTTPBasicAuth', 'client.py', `requests.get(url, auth=HTTPBasicAuth('alice', '${password}'))\n`], + ['Java Credentials.basic', 'Client.java', `String c = Credentials.basic("alice", "${password}");\n`], + ['Java new Basic', 'Client.java', `Auth a = new Basic("alice", "${password}");\n`], + ['Go SetBasicAuth', 'client.go', `req.SetBasicAuth("alice", "${password}")\n`], + ['C# NetworkCredential', 'Client.cs', `var c = new NetworkCredential("alice", "${password}");\n`], + ['supertest .auth', 'api.test.js', `await request(app).get('/x').auth('alice', '${password}');\n`], + ]; + it.each(literal)('reports: %s', (_label, file, text) => { + expect(rules(file, text)).toContain('http-auth-credential'); + }); + + // The sibling credential headers and their names were already secret-like; they must stay reported. + it.each(['X-API-Key', 'X-Api-Key', 'Api-Key', 'X-Auth-Token', 'X-Access-Token', 'Ocp-Apim-Subscription-Key', 'X-Amz-Security-Token', 'PRIVATE-TOKEN'])( + 'reports the sibling header %s in curl, source and YAML', + (header) => { + expect(count('run.sh', `curl -H "${header}: ${token}" https://api.internal.corp\n`)).toBe(1); + expect(count('src/api.js', `fetch(u, { headers: { '${header}': '${token}' } });\n`)).toBe(1); + expect(count('ci.yml', `${header}: ${token}\n`)).toBe(1); + }, + ); + + const clean = [ + ['${...} in a template literal', 'src/api.js', 'fetch(u, { headers: { Authorization: `Bearer ${token}` } });\n'], + ['$TOKEN', 'run.sh', `curl -H "${AUTH}: Bearer $TOKEN" https://api.internal.corp\n`], + ['${TOKEN}', 'run.sh', `curl -H "${AUTH}: Bearer \${TOKEN}" https://api.internal.corp\n`], + ['', 'README.md', `${AUTH}: Bearer \n`], + ['', 'README.md', `${AUTH}: Bearer \n`], + ['{{ token }}', 'ci.yml', `${AUTH}: Bearer {{ token }}\n`], + ['{token}', 'client.py', `requests.get(u, headers={"${AUTH}": f"Bearer {self.access_token_value}"})\n`], + ['xxx', 'README.md', `${AUTH}: Bearer xxxxxxxxxxxxxxxx\n`], + ['ellipsis', 'README.md', `${AUTH}: Bearer eyJhbGciOi...\n`], + ['YOUR_ prefix', 'README.md', `curl -H "${AUTH}: Bearer YOUR_FIREBASE_ID_TOKEN" http://localhost:3000/api/x\n`], + ['scheme only', 'README.md', `${AUTH}: Bearer\n`], + ['prose', 'README.md', `The ${AUTH} header carries the credentials of the client.\n`], + ['concatenation', 'src/api.js', `fetch(u, { headers: { ${AUTH}: 'Bearer ' + accessTokenValueFromStore } });\n`], + ['process.env', 'src/api.js', `fetch(u, { headers: { ${AUTH}: 'Bearer ' + process.env.API_TOKEN } });\n`], + ['shorthand identifier', 'src/api.js', `fetch(u, { headers: { ${AUTH}: authorizationHeaderValue } });\n`], + ['function call', 'src/api.js', `res.headers.set('${AUTH}', getAuthorizationHeaderValue());\n`], + ['CORS allow-list', 'src/cors.js', `res.setHeader('Access-Control-Allow-Headers', 'Content-Type, ${AUTH}, X-Requested-With');\n`], + ['CORS allow-list array', 'src/cors.js', `allowedHeaders: ['Content-Type', '${AUTH}', 'X-Requested-With'],\n`], + ['CORS allow-list YAML', 'cors.yml', `allow_headers: Content-Type, ${AUTH}, X-Requested-With\n`], + ['Vary', 'src/cors.js', `res.setHeader('Vary', '${AUTH}');\n`], + ['WWW-Authenticate challenge', 'src/api.js', `res.setHeader('WWW-Authenticate', 'Bearer realm="api", error="invalid_token"');\n`], + ['bearer test value', 'api.test.js', `request(app).get('/x').set('${AUTH}', 'Bearer invalid-token');\n`], + ['Basic user:pass', 'README.md', `${AUTH}: Basic ${placeholderBasic}\n`], + ['Basic user:password', 'README.md', `${AUTH}: Basic ${Buffer.from('username:password').toString('base64')}\n`], + ['Basic ', 'README.md', `${AUTH}: Basic \n`], + ['JWT (the jwt-token rule reports it, not this one)', 'README.md', `${AUTH}: Bearer ${base64url({ alg: 'none' })}.${base64url({ sub: 'x' })}.\n`], + ['nginx variable', 'nginx.conf', `proxy_set_header ${AUTH} $http_authorization;\n`], + ['nginx variable, quoted', 'nginx.conf', `proxy_set_header ${AUTH} "$http_authorization";\n`], + ['Python auth from variables', 'client.py', 'requests.get(url, auth=(user, password))\n'], + ['Python auth placeholders', 'client.py', 'requests.get(url, auth=("user", "pass"))\n'], + ['Python auth from the environment', 'client.py', 'requests.get(url, auth=("alice", os.environ["PW"]))\n'], + ['Java Credentials.basic from variables', 'Client.java', 'String c = Credentials.basic(user, pw);\n'], + ['Digest without a response', 'README.md', `${AUTH}: Digest username="alice", realm="api"\n`], + ['X-API-Key placeholder', 'run.sh', 'curl -H "X-API-Key: YOUR_API_KEY" https://api.internal.corp\n'], + ['X-API-Key variable', 'run.sh', 'curl -H "X-API-Key: $API_KEY" https://api.internal.corp\n'], + ]; + it.each(clean)('passes: %s', (_label, file, text) => { + expect(count(file, text)).toBe(0); + }); + + it('a JWT is reported once, by the jwt-token rule', () => { + const jwt = [base64url({ alg: 'HS256', typ: 'JWT' }), base64url({ sub: 'user-1234', role: 'admin' }), randomString(43, 13005, B64URL)].join('.'); + expect(rules('notes.txt', `${AUTH}: Bearer ${jwt}\n`)).toEqual(['jwt-token']); + }); + + it('Basic credentials are decoded: a real password is reported, a placeholder or the RFC sample is not', () => { + expect(count('notes.txt', `${AUTH}: Basic ${basic}\n`)).toBe(1); + expect(count('notes.txt', `${AUTH}: Basic ${Buffer.from(['Aladdin', 'open sesame'].join(':')).toString('base64')}\n`)).toBe(0); + // a Basic value that is not user:password is judged as an opaque token + expect(count('notes.txt', `${AUTH}: Basic ${randomString(30, 13007)}\n`)).toBe(1); + }); + + it('the allow marker silences a header line', () => { + expect(count('notes.txt', `${AUTH}: Bearer ${token} # ${ALLOW_MARKER}\n`)).toBe(0); + }); + + it('is reported by the tree scan, --range and --history without printing the value', SLOW, () => { + expectReported('http-auth-credential', 'docs/api.md', `${AUTH}: Bearer \n`, `${AUTH}: Bearer ${token}\n`, token); + expectReported('http-auth-credential', 'src/api.js', "fetch(u, { headers: { Authorization: 'Bearer ' + t } });\n", `fetch(u, { headers: { Authorization: 'Bearer ${token}' } });\n`, token); + }); + + it('hostile header text is scanned in linear time', SLOW, () => { + const timings = timeInChild(` + const H = 'Author' + 'ization'; + const N = 20000; + const texts = [ + H + ':' + ' '.repeat(200000) + 'x', 'password:' + ' '.repeat(200000) + 'x', 'foo:' + ' '.repeat(200000) + 'x', (H + ': Bearer ').repeat(N), ("'" + H + "'").repeat(N), H.repeat(N), 'A-'.repeat(100000) + H, + H + ': Bearer ' + 'a'.repeat(300000), H + ': ' + 'a-'.repeat(50000) + ' x', H + ': Digest ' + 'a '.repeat(100000) + 'response=', + (H + ': Digest ').repeat(N), 'auth=(' + 'a,'.repeat(100000), 'auth = ('.repeat(N), 'BasicAuth('.repeat(N), 'proxy_set_header '.repeat(N), + ]; + for (const file of ['a.md', 'a.js', 'nginx.conf', 'ci.yml']) for (const text of texts) { + const started = performance.now(); + scanText(file, text); + timings.push(Math.round(performance.now() - started)); + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + // ------------------------------------------------------------------------- + describe('(2) Terraform variable labels and the other ways a variable is set', () => { + const NAMES = ['api_' + 'token', 'db_' + 'password', 'client_' + 'secret', 'private_' + 'key', 'admin_' + 'pass']; + const variable = (name, body) => `variable "${name}" {\n${body}}\n`; + const defaultOf = (v) => ` type = string\n default = ${v}\n`; + + it.each(NAMES)('reports a literal default of variable "%s"', (name) => { + expect(rules('variables.tf', variable(name, defaultOf(`"${token}"`)))).toEqual(['secret-name-value-pair']); + expect(rules('main.tf', variable(name, ` default = "${token}"\n sensitive = true\n`))).toEqual(['secret-name-value-pair']); + expect(rules('main.tf', `variable "${name}" { default = "${token}" }\n`)).toEqual(['secret-name-value-pair']); + }); + + it('reports the block forms and the file kinds that hold them', () => { + const name = NAMES[0]; + expect(count('vars.hcl', variable(name, defaultOf(`"${token}"`)))).toBe(1); + expect(count('build.pkr.hcl', variable(name, defaultOf(`"${token}"`)))).toBe(1); + expect(count('README.md', `\`\`\`hcl\n${variable(name, defaultOf(`"${token}"`))}\`\`\`\n`)).toBe(1); + expect(count('main.tf', `output "${name}" {\n value = "${token}"\n}\n`)).toBe(1); + expect(count('main.tf', `variable "${name}" {\n default = < 3\n error_message = "too short"\n }\n default = "${token}"\n}\n`)).toBe(1); + // already found before: the plain assignments the label form is a sibling of + expect(count('main.tf', `locals {\n ${name} = "${token}"\n}\n`)).toBe(1); + expect(count('main.tf', `resource "aws_db_instance" "x" {\n password = "${password}${password}"\n}\n`)).toBe(1); + expect(count('prod.tfvars', `${name} = "${token}"\n`)).toBe(1); + }); + + const clean = [ + ['no default', variable(NAMES[0], ' type = string\n sensitive = true\n')], + ['description and sensitive only', variable(NAMES[0], ' description = "The API token for the service"\n type = string\n sensitive = true\n')], + ['empty string', variable(NAMES[0], defaultOf('""'))], + ['null', variable(NAMES[0], defaultOf('null'))], + ['var reference', variable(NAMES[0], defaultOf('var.other'))], + ['data reference', variable(NAMES[0], defaultOf('data.aws_ssm_parameter.x.value'))], + ['file() call', variable(NAMES[0], defaultOf('file("token.txt")'))], + ['interpolation', variable(NAMES[0], defaultOf('"${var.prefix}-token"'))], + ['placeholder', variable(NAMES[0], defaultOf('"changeme"'))], + ['marker', variable(NAMES[0], defaultOf('""'))], + ['empty list', variable('api_' + 'tokens', defaultOf('[]'))], + ['empty map', variable(NAMES[0], defaultOf('{}'))], + ['a name that is not secret-like', variable('region', defaultOf(`"${token}"`))], + ['a URL under a weak name', variable('api_token_url', defaultOf('"https://api.internal.corp/token"'))], + ['a TTL under a weak name', variable('token_ttl', defaultOf('"3600"'))], + ['a description that mentions a token', variable('region', ` description = "Token bucket region"\n default = "${token}"\n`).replace(token, 'us-east-1')], + ['the next block belongs to another variable', `${variable(NAMES[0], ' type = string\n')}\n${variable('region', defaultOf('"us-east-1"'))}`], + ['a secret-named variable followed by a plain resource', `${variable('db_' + 'password', ' type = string\n')}resource "x" "y" {\n password = var.db_password\n}\n`], + ]; + it.each(clean)('passes: %s', (_label, text) => { + expect(count('main.tf', text)).toBe(0); + }); + + it('a variable label in source code is not a Terraform block', () => { + expect(count('src/notes.js', `// variable "${NAMES[0]}" { default = "${token}" }\n`)).toBe(0); + }); + + it('-var, --var, TF_VAR_ and --build-arg forms are reported when they hold a literal, and pass for references', () => { + const name = NAMES[0]; + for (const line of [ + `terraform apply -var ${name}=${token}\n`, + `terraform apply --var '${name}=${token}'\n`, + `terraform plan -var "${name}=${token}" -var region=us-east-1\n`, + `tofu apply -var ${name}=${token}\n`, + `export TF_VAR_${name}=${token}\n`, + `TF_VAR_${name}=${token} terraform apply\n`, + `docker build --build-arg ${name.toUpperCase()}=${token} .\n`, + ]) expect(count('run.sh', line), line.slice(0, 40)).toBe(1); + expect(count('.env', `TF_VAR_${name}=${token}\n`)).toBe(1); + expect(count('ci.yml', `env:\n TF_VAR_${name}: ${token}\n`)).toBe(1); + for (const line of [ + `terraform apply -var ${name}=$TOKEN\n`, + `terraform apply -var "${name}=\${TOKEN}"\n`, + `terraform apply -var region=${token}\n`, + 'terraform apply -var-file=secrets.tfvars\n', + `docker build --build-arg ${name.toUpperCase()} .\n`, + `export TF_VAR_${name}=$API_TOKEN\n`, + ]) expect(count('run.sh', line), line.slice(0, 40)).toBe(0); + }); + + it('pulumi config set: a positional value under a secret-like name, or any name with --secret', () => { + for (const line of [ + `pulumi config set --secret dbPassword ${password}\n`, + `pulumi config set apiToken ${token} --secret\n`, + `pulumi config set app:apiToken ${token}\n`, + `pulumi config set --secret region ${token}\n`, + ]) expect(rules('deploy.sh', line), line.slice(0, 40)).toEqual(['secret-cli-command']); + // (assembled from words: a JS string that ends a quoted argument with its own quote would read as one unterminated shell word) + const pulumi = (...args) => ['pulumi', 'config', ...args].join(' '); + for (const line of [pulumi('set', '--secret', 'dbPassword', '"$DB_PASSWORD"'), pulumi('set', 'region', 'us-west-2'), pulumi('set-all', '--secret', 'x')]) { + expect(count('deploy.sh', `${line}\n`), line.slice(0, 40)).toBe(0); + } + }); + + it('Pulumi..yaml: a plaintext config value is reported, the secure ciphertext is not', () => { + expect(rules('Pulumi.dev.yaml', `config:\n app:apiToken: ${token}\n aws:region: us-west-2\n`)).toEqual(['secret-assignment']); + expect(rules('infra/Pulumi.yaml', `config:\n app:dbPassword: "${password}${password}"\n`)).toEqual(['secret-assignment']); + expect(count('Pulumi.dev.yaml', `config:\n app:apiToken:\n secure: v1:${randomString(12, 13008)}:${randomString(60, 13009, BASE64)}\n`)).toBe(0); + expect(count('Pulumi.dev.yaml', 'config:\n app:apiToken: ${TOKEN}\n')).toBe(0); + // a namespaced key outside a Pulumi file is not this format + expect(count('other.yaml', `config:\n app:region: ${token}\n`)).toBe(0); + }); + + it('a kustomize secretGenerator literal is reported', () => { + expect(count('kustomization.yaml', `secretGenerator:\n- name: s\n literals:\n - ${NAMES[0]}=${token}\n`)).toBe(1); + expect(count('kustomization.yaml', `secretGenerator:\n - name: s\n literals:\n - "${NAMES[1]}=${password}${password}"\n`)).toBe(1); + }); + + it('an ansible-vault ciphertext is not a plaintext credential, a plaintext vars value is', () => { + const header = ['$ANSIBLE', '_VAULT;1.1;AES256'].join(''); + const body = randomString(64, 13010, HEX); + expect(count('play.yml', `vars:\n ${NAMES[0]}: !vault |\n ${header}\n ${body}\n`)).toBe(0); + expect(count('play.yml', `vars:\n ${NAMES[1]}: |\n ${header}\n ${body}\n ${body}\n`)).toBe(0); + expect(count('play.yml', `vars:\n ${NAMES[0]}: ${token}\n`)).toBe(1); + }); + + it('the value line of a Terraform default is what --range and --history blame', SLOW, () => { + // the label line is unchanged in the second commit: only the default line is added + const name = NAMES[0]; + expectReported('secret-name-value-pair', 'variables.tf', variable(name, defaultOf('"changeme"')), variable(name, defaultOf(`"${token}"`)), token); + // an allow marker on the default line silences it, and one on an unrelated line does not + expect(count('variables.tf', variable(name, ` type = string\n default = "${token}" # ${ALLOW_MARKER}\n`))).toBe(0); + expect(count('variables.tf', `# ${ALLOW_MARKER}\n${variable(name, defaultOf(`"${token}"`))}`)).toBe(1); + }); + + it('many labels and long blocks are read within a budget', SLOW, () => { + const timings = timeInChild(` + const texts = ['variable "api_' + 'token" '.repeat(20000), 'variable "api_' + 'token" {\\n'.repeat(4000), 'variable "' + 'a'.repeat(200000), + 'variable "api_' + 'token" {\\n' + ' x = 1\\n'.repeat(40000), 'a:'.repeat(100000)]; + for (const file of ['main.tf', 'README.md', 'Pulumi.dev.yaml']) for (const text of texts) { + const started = performance.now(); + scanText(file, text); + timings.push(Math.round(performance.now() - started)); + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + // ------------------------------------------------------------------------- + describe('(3) whitespace-delimited service configuration', () => { + const REQUIREPASS = ['require', 'pass'].join(''); + const MASTERAUTH = ['master', 'auth'].join(''); + const pw = randomString(16, 13011); + const digest = randomString(64, 13012, HEX); + + const literal = [ + ['redis requirepass', 'redis.conf', `${REQUIREPASS} ${pw}\n`], + ['redis masterauth', 'redis.conf', `${MASTERAUTH} ${pw}\n`], + ['redis requirepass, quoted with spaces', 'redis.conf', `${REQUIREPASS} "${pw} ${pw}"\n`], + ['redis requirepass in a numbered file', 'redis-6379.conf', `${REQUIREPASS} ${pw}\n`], + ['redis requirepass in a template', 'conf/redis.conf.j2', `${REQUIREPASS} ${pw}\n`], + ['redis requirepass in a ConfigMap block', 'configmap.yaml', `data:\n redis.conf: |\n ${REQUIREPASS} ${pw}\n`], + ['redis requirepass in a Markdown fence', 'README.md', `\`\`\`\n${REQUIREPASS} ${pw}\n\`\`\`\n`], + ['redis tls-key-file-pass', 'redis.conf', `tls-key-file-pass ${pw}\n`], + ['redis tls-client-key-file-pass', 'redis.conf', `tls-client-key-file-pass ${pw}\n`], + ['redis-server option', 'run.sh', `redis-server --${REQUIREPASS} ${pw}\n`], + ['redis-server option, compose list', 'docker-compose.yml', `command: ["redis-server", "--${REQUIREPASS}", "${pw}"]\n`], + ['redis CONFIG SET', 'run.sh', `redis-cli config set ${REQUIREPASS} ${pw}\n`], + ['redis CONFIG SET, in a fence', 'README.md', `\`\`\`\nCONFIG SET ${REQUIREPASS} ${pw}\n\`\`\`\n`], + ['redis-cli -a', 'run.sh', `redis-cli -a ${pw} ping\n`], + ['redis-cli REDISCLI_AUTH', 'run.sh', `REDISCLI_AUTH=${pw} redis-cli ping\n`], + ['redis ACL plaintext password', 'redis.conf', `user default on >${pw} ~* +@all\n`], + ['redis ACL password, other user', 'users.acl', `user app on >${pw} ~app:* +get\n`], + ['redis ACL password hash', 'redis.conf', `user default on #${digest} ~* +@all\n`], + ['redis ACL SETUSER', 'README.md', `\`\`\`\nACL SETUSER app on >${pw} ~* +@all\n\`\`\`\n`], + ['sentinel auth-pass', 'sentinel.conf', `sentinel auth-pass mymaster ${pw}\n`], + ['sentinel auth-pass, redis-sentinel.conf', 'conf/redis-sentinel.conf', `sentinel auth-pass mymaster ${pw}\n`], + ['mosquitto password', 'mosquitto.conf', `password ${pw}\n`], + ['mosquitto bridge_password', 'mosquitto.conf', `connection b\nbridge_password ${pw}\n`], + ['mosquitto remote_password', 'mosquitto.conf', `remote_password ${pw}\n`], + ['mosquitto_passwd -b', 'setup.sh', `mosquitto_passwd -b /etc/mosquitto/passwd alice ${pw}\n`], + ['mosquitto_pub -P', 'setup.sh', `mosquitto_pub -h h -t t -m hi -u alice -P ${pw}\n`], + ['htpasswd -b', 'setup.sh', `htpasswd -b .htpasswd alice ${pw}\n`], + ['htpasswd -bc', 'setup.sh', `htpasswd -bc .htpasswd alice ${pw}\n`], + ['htpasswd -nb', 'setup.sh', `htpasswd -nb alice ${pw}\n`], + ['msmtprc password', 'msmtprc', `account default\nhost smtp.internal.corp\npassword ${pw}\n`], + ['.msmtprc password', '.msmtprc', `password ${pw}\n`], + ['.fetchmailrc password', '.fetchmailrc', `poll mail.internal.corp protocol pop3 user "alice" password "${pw}"\n`], + ['.fetchmailrc with password', '.fetchmailrc', `poll mail.internal.corp protocol imap user alice there with password ${pw} is alice here\n`], + ['haproxy userlist insecure-password', 'haproxy.cfg', `userlist L\n user admin insecure-password ${pw}\n`], + ['haproxy userlist password', 'haproxy.cfg', `userlist L\n user admin password ${pw}\n`], + ['haproxy stats auth', 'haproxy.cfg', `listen stats\n stats auth admin:${pw}\n`], + ['haproxy set-header', 'haproxy.cfg', `backend b\n http-request set-header X-Api-Key ${token}\n`], + ['nginx proxy_set_header', 'nginx.conf', `proxy_set_header X-Api-Key "${token}";\n`], + ['nginx fastcgi_param', 'site.conf', `fastcgi_param DB_PASSWORD "${pw}";\n`], + ['nginx set variable', 'nginx.conf', `set $api_token "${token}";\n`], + ['Apache SetEnv', 'httpd.conf', `SetEnv DB_PASSWORD ${pw}\n`], + ['Apache SetEnv, quoted', 'site.conf', `\n SetEnv API_TOKEN "${token}"\n\n`], + ['Apache SetEnv in .htaccess', '.htaccess', `SetEnv DB_PASSWORD ${pw}\n`], + ['tinyproxy BasicAuth', 'tinyproxy.conf', `BasicAuth alice ${pw}\n`], + ['postgresql ssl_passphrase_command', 'postgresql.conf', `ssl_passphrase_command = 'echo ${pw}'\n`], + ['rabbitmq advanced.config', 'advanced.config', `[{rabbit,[{default_pass, <<"${pw}">>}]}].\n`], + ['rabbitmq.conf', 'rabbitmq.conf', `default_pass = ${pw}\n`], + ['rabbitmqctl add_user', 'setup.sh', `rabbitmqctl add_user alice ${pw}\n`], + ['rabbitmqctl change_password', 'setup.sh', `rabbitmqctl change_password alice ${pw}\n`], + ['rabbitmqadmin -p', 'setup.sh', `rabbitmqadmin -u alice -p ${pw} list queues\n`], + ['openvpn inline credentials', 'client.ovpn', `\nalice\n${pw}\n\n`], + ['dovecot SQL connect', 'dovecot-sql.conf.ext', `connect = host=db.internal.corp dbname=mail user=mail password=${pw}\n`], + ['dovecot LDAP dnpass', 'dovecot-ldap.conf.ext', `dnpass = ${pw}\n`], + ['sshpass -p', 'run.sh', `sshpass -p ${pw} ssh alice@h\n`], + ['SSHPASS variable', 'run.sh', `SSHPASS=${pw} sshpass -e ssh h\n`], + ['docker login -p', 'run.sh', `docker login -u alice -p ${pw}\n`], + ['az login -p', 'run.sh', `az login -u alice -p ${pw}\n`], + ['sqlcmd -P', 'run.sh', `sqlcmd -S h -U sa -P ${pw}\n`], + ['keytool -storepass', 'run.sh', `keytool -genkey -storepass ${pw} -keypass ${pw}\n`], + ['gpg --passphrase', 'run.sh', `gpg --batch --passphrase ${pw} -d f.gpg\n`], + ['vault login', 'run.sh', `vault login ${token}\n`], + ['chpasswd from echo', 'Dockerfile', `RUN echo 'root:${pw}' | chpasswd\n`], + ['chpasswd from a here-string', 'run.sh', `chpasswd <<< "alice:${pw}"\n`], + ['aws configure set', 'run.sh', `aws configure set aws_secret_access_key ${token}\n`], + ]; + it.each(literal)('reports: %s', (_label, file, text) => { + expect(count(file, text)).toBeGreaterThan(0); + }); + + it('names the rule that owns each directive family', () => { + expect(rules('redis.conf', `${REQUIREPASS} ${pw}\n`)).toEqual(['config-directive-secret']); + expect(rules('haproxy.cfg', `userlist L\n user admin insecure-password ${pw}\n`)).toEqual(['config-directive-secret']); + expect(rules('nginx.conf', `proxy_set_header ${AUTH} "Bearer ${token}";\n`)).toEqual(['http-auth-credential']); + }); + + const clean = [ + ['requirepass with no value', 'redis.conf', `${REQUIREPASS}\n`], + ['requirepass empty string', 'redis.conf', `${REQUIREPASS} ""\n`], + ['requirepass commented out', 'redis.conf', `# ${REQUIREPASS} ${pw}\n`], + ['requirepass placeholder', 'redis.conf', `${REQUIREPASS} changeme\n`], + ['requirepass marker', 'redis.conf', `${REQUIREPASS} \n`], + ['requirepass environment reference', 'redis.conf', `${REQUIREPASS} \${REDIS_PASSWORD}\n`], + ['the documented redis.conf sample', 'redis.conf', `${REQUIREPASS} foobared\n`], + ['masteruser is a user name', 'redis.conf', `masteruser ${pw}\n`], + ['sentinel auth-user is a user name', 'sentinel.conf', `sentinel auth-user mymaster ${pw}\n`], + ['ACL nopass', 'redis.conf', 'user default on nopass ~* +@all\n'], + ['ACL disabled user', 'redis.conf', 'user default off\n'], + ['ACL placeholder password', 'redis.conf', 'user default on >changeme ~* +@all\n'], + ['nginx user directive', 'nginx.conf', 'user www-data;\n'], + ['tls-key-file is a path', 'redis.conf', 'tls-key-file /etc/redis/key.pem\n'], + ['requirepass in prose', 'README.md', `Set ${REQUIREPASS} in redis.conf to protect the server. The ${REQUIREPASS} directive is documented.\n`], + ['requirepass in a code file', 'src/notes.js', `// ${REQUIREPASS} ${pw}\n`], + ['redis-server option with a variable', 'run.sh', `redis-server --${REQUIREPASS} "$REDIS_PASSWORD"\n`], + ['redis-cli with a variable', 'run.sh', 'redis-cli -a "$REDIS_PASSWORD" ping\n'], + ['mosquitto password_file is a path', 'mosquitto.conf', 'password_file /etc/mosquitto/conf.d/a-rather-long-file-of-passwords.txt\n'], + ['mosquitto_passwd without -b prompts', 'setup.sh', 'mosquitto_passwd -c /etc/mosquitto/passwd alice\n'], + ['mosquitto_passwd with a variable', 'setup.sh', 'mosquitto_passwd -b /etc/mosquitto/passwd alice "$PW"\n'], + ['mosquitto_sub with a variable', 'setup.sh', 'mosquitto_sub -h h -t t -u alice -P "$PW"\n'], + ['htpasswd without -b prompts', 'setup.sh', 'htpasswd -c .htpasswd alice\n'], + ['htpasswd with a variable', 'setup.sh', 'htpasswd -b .htpasswd alice "$PW"\n'], + ['msmtp passwordeval runs a command', 'msmtprc', 'passwordeval "pass show mail"\n'], + ['msmtp empty password', '.msmtprc', 'password \n'], + ['fetchmail password from a variable', '.fetchmailrc', 'poll mail.internal.corp user alice password "$PW"\n'], + ['fetchmail empty password', '.fetchmailrc', 'poll mail.internal.corp proto imap user alice password ""\n'], + ['haproxy hashed password', 'haproxy.cfg', `userlist L\n user admin password $6$rounds=1$${randomString(8, 13013)}$${randomString(40, 13014)}\n`], + ['haproxy user outside a userlist', 'haproxy.cfg', `global\n user haproxy password ${pw}\n`], + ['haproxy stats auth placeholder', 'haproxy.cfg', 'listen stats\n stats auth admin:changeme\n'], + ['haproxy server line', 'haproxy.cfg', 'backend b\n server s1 10.0.0.1:80 check\n'], + ['haproxy header from a fetch', 'haproxy.cfg', 'backend b\n http-request set-header X-Api-Key %[req.hdr(x)]\n'], + ['nginx header from a variable', 'nginx.conf', 'proxy_set_header X-Api-Key $http_x_api_key;\n'], + ['nginx header with a name that is no credential', 'nginx.conf', `proxy_set_header X-Request-Id ${token};\n`], + ['nginx ssl_password_file is a path', 'nginx.conf', 'ssl_password_file /etc/nginx/pass.txt;\n'], + ['nginx auth_basic_user_file is a path', 'nginx.conf', 'auth_basic_user_file /etc/nginx/.htpasswd;\n'], + ['nginx set of a plain variable', 'nginx.conf', `set $upstream_name "${token}";\n`], + ['Apache SetEnv of a plain variable', 'httpd.conf', 'SetEnv APP_ENV production\n'], + ['Apache SetEnv from a variable', 'httpd.conf', 'SetEnv DB_PASSWORD $DB_PASSWORD\n'], + ['Apache PassEnv', 'httpd.conf', 'PassEnv DB_PASSWORD\n'], + ['Apache AuthUserFile is a path', 'httpd.conf', 'AuthUserFile /etc/apache2/.htpasswd\n'], + ['openvpn auth-user-pass names a file', 'client.ovpn', 'auth-user-pass /etc/openvpn/creds\n'], + ['openvpn auth-user-pass prompts', 'client.ovpn', 'auth-user-pass\n'], + ['postgresql ssl_passphrase_command runs a script', 'postgresql.conf', "ssl_passphrase_command = '/usr/local/bin/get-passphrase'\n"], + ['postgresql password_encryption', 'postgresql.conf', 'password_encryption = scram-sha-256\n'], + ['mongod keyFile is a path', 'mongod.conf', 'security:\n keyFile: /etc/mongo/keyfile\n'], + ['rabbitmq default_pass from a variable', 'rabbitmq.conf', 'default_pass = ${RABBIT_PW}\n'], + ['rabbitmq advanced.config, another tuple', 'advanced.config', '[{rabbit,[{default_user, <<"guest">>}]}].\n'], + ['git credential helper store', '.gitconfig', '[credential]\n helper = store\n'], + ['sshpass -f reads a file', 'run.sh', 'sshpass -f /run/secrets/pw ssh alice@h\n'], + ['sshpass -e reads the environment', 'run.sh', 'sshpass -e ssh alice@h\n'], + ['sshpass with a variable', 'run.sh', 'sshpass -p "$PW" ssh alice@h\n'], + ['docker login --password-stdin', 'run.sh', 'echo "$PW" | docker login -u alice --password-stdin\n'], + ['docker login with a variable', 'run.sh', 'docker login -u alice -p $pw\n'], + ['sqlcmd with a variable', 'run.sh', 'sqlcmd -S h -U sa -P "$PW"\n'], + ['keytool with the documented default', 'run.sh', 'keytool -list -storepass changeit\n'], + ['vault login prompts', 'run.sh', 'vault login -method=oidc\n'], + ['vault login with a variable', 'run.sh', 'vault login "$VAULT_TOKEN"\n'], + ['chpasswd with a variable', 'Dockerfile', 'RUN echo "root:$ROOT_PW" | chpasswd\n'], + ['aws configure set of a plain setting', 'run.sh', `aws configure set region ${token}\n`], + ]; + it.each(clean)('passes: %s', (_label, file, text) => { + expect(count(file, text)).toBe(0); + }); + + it('whole-file secrets: an Erlang cookie and a MongoDB key file', () => { + expect(credentialFormats('.erlang.cookie').has('vault')).toBe(true); + expect(credentialFormats('etc/mongodb-keyfile').has('vault')).toBe(true); + expect(credentialFormats('keyfile').has('vault')).toBe(true); + expect(count('.erlang.cookie', `${randomString(20, 13015, UPPER)}\n`)).toBe(1); + expect(count('mongodb-keyfile', `${randomString(60, 13016, BASE64)}\n${randomString(60, 13017, BASE64)}\n`)).toBe(2); + expect(count('.erlang.cookie', 'your_cookie_here\n')).toBe(0); + expect(credentialFormats('keyfile.md').has('vault')).toBe(false); + }); + + it('is reported by the tree scan, --range and --history without printing the value', SLOW, () => { + expectReported('config-directive-secret', 'conf/redis.conf', `${REQUIREPASS} changeme\nport 6379\n`, `${REQUIREPASS} ${pw}\nport 6379\n`, pw); + expectReported('config-directive-secret', 'conf/redis.conf', 'user default on nopass ~* +@all\n', `user default on >${pw} ~* +@all\n`, pw); + expectReported('config-directive-secret', 'haproxy.cfg', 'userlist L\n user admin insecure-password changeme\n', `userlist L\n user admin insecure-password ${pw}\n`, pw); + }); + + it('the allow marker silences a directive line', () => { + expect(count('redis.conf', `${REQUIREPASS} ${pw} # ${ALLOW_MARKER}\n`)).toBe(0); + expect(count('redis.conf', `# ${ALLOW_MARKER}\n${REQUIREPASS} ${pw}\n`)).toBe(1); + }); + + it('hostile directive text is scanned in linear time', SLOW, () => { + const timings = timeInChild(` + const N = 20000; + const texts = [ + 'requirepass '.repeat(N), 'requirepass "' + 'a\\\\'.repeat(100000), 'requirepass \\n'.repeat(N), 'user x ' + '>a '.repeat(100000), + 'user x on\\n'.repeat(N), 'sentinel auth-pass ' + 'a '.repeat(100000), ('a'.repeat(100) + ' ').repeat(3000), + ('password_a ' + 'b'.repeat(200) + '\\n').repeat(2000), 'userlist\\n' + 'user a '.repeat(N), 'proxy_set_header '.repeat(N), + 'proxy_set_header ' + 'a'.repeat(200000), 'set $' + 'a'.repeat(200000), "ssl_passphrase_command = '" + 'echo '.repeat(N), + '{a,'.repeat(N) + '"', '{' + ' '.repeat(100000) + 'a', '\\n' + 'a\\n'.repeat(N), + 'echo ' + 'a:'.repeat(50000) + ' | chpasswd', 'chpasswd <<< ' + 'a:'.repeat(100000), 'docker login ' + '-p '.repeat(N), + ]; + for (const file of ['a.md', 'a.sh', 'redis.conf', 'mosquitto.conf', 'haproxy.cfg', 'advanced.config', 'nginx.conf']) for (const text of texts) { + const started = performance.now(); + scanText(file, text); + timings.push(Math.round(performance.now() - started)); + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); +}); + +// --------------------------------------------------------------------------- +// Review round 14: quoted passphrases in Basic-auth calls, merge results, PEM blocks in lockfile history. +// Every value is generated or assembled at run time; the names that would look like secrets are split. +// --------------------------------------------------------------------------- +describe('review round 14', () => { + const phrase = ['correct horse', 'battery staple'].join(' '); + const random = randomString(20, 14001); + const count = (file, text) => scanText(file, text).length; + + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + const run = (cmd, args, cwd) => spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: SLOW_TEST_MS, maxBuffer: 64 * 1024 * 1024 }); + const scan = (cwd, ...args) => run(process.execPath, [SCANNER, ...args], cwd); + const git = (cwd, ...args) => run('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', '-c', 'commit.gpgsign=false', ...args], cwd); + const makeRepo = () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-r14-')); + dirs.push(dir); + expect(run('git', ['init', '-q', '-b', 'main'], dir).status).toBe(0); + return dir; + }; + const commit = (dir, files) => { + for (const [file, content] of Object.entries(files)) { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), content); + } + git(dir, 'add', '-A'); + expect(git(dir, 'commit', '-q', '-m', 'c').status).toBe(0); + return git(dir, 'rev-parse', 'HEAD').stdout.trim(); + }; + const branchFrom = (dir, name, revision) => expect(git(dir, 'checkout', '-q', '-b', name, revision).status).toBe(0); + + // ------------------------------------------------------------------------- + describe('(1) a quoted passphrase with spaces in a Basic-auth call is read whole', () => { + // The call names are split, so this file does not spell a call the scanner looks for. + const HTTP_BASIC = ['HTTP', 'BasicAuth'].join(''); + const CREDS_BASIC = ['Credentials', '.basic'].join(''); + const NEW_BASIC = ['new ', 'Basic'].join(''); + const NET_CRED = ['Network', 'Credential'].join(''); + const SET_BASIC = ['Set', 'BasicAuth'].join(''); + const AUTH_KW = ['au', 'th'].join(''); + const forms = [ + ['a.py', (q) => `requests.get(url, ${AUTH_KW}=("admin", ${q}))\n`], + ['a.py', (q) => `${HTTP_BASIC}("admin", ${q})\n`], + ['a.js', (q) => `request(app).get('/x').${AUTH_KW}('admin', ${q})\n`], + ['a.js', (q) => `const agent = request(app); agent.${AUTH_KW}('admin', ${q});\n`], // a name right before the dot + ['A.java', (q) => `${CREDS_BASIC}("admin", ${q})\n`], + ['A.java', (q) => `${NEW_BASIC}("admin", ${q})\n`], + ['A.cs', (q) => `new ${NET_CRED}("admin", ${q})\n`], + ['a.go', (q) => `req.${SET_BASIC}("admin", ${q})\n`], + ['a.rb', (q) => `req.basic_auth("admin", ${q})\n`], + ['a.php', (q) => `Http::withBasicAuth('admin', ${q})\n`], + ]; + const quotings = [ + ['double quotes', (t) => `"${t}"`], + ['single quotes', (t) => `'${t}'`], + ['backticks', (t) => `\`${t}\``], + ['triple double quotes', (t) => `"""${t}"""`], + ['triple single quotes', (t) => `'''${t}'''`], + ]; + + for (const [quoting, quote] of quotings) { + it.each(forms)(`reports the passphrase in ${quoting}: %s`, (file, form) => { + expect(count(file, form(quote(phrase)))).toBe(1); + }); + } + + it('reads an escaped quote inside the passphrase as part of it', () => { + expect(count('a.py', `${HTTP_BASIC}("admin", "correct \\"horse\\" battery staple")\n`)).toBe(1); + expect(count('a.js', `agent.${AUTH_KW}('admin', 'correct \\'horse\\' battery staple')\n`)).toBe(1); + }); + + it('still reports a single random word', () => { + expect(count('a.py', `${HTTP_BASIC}("admin", "${random}")\n`)).toBe(1); + }); + + it('judges a passphrase like every other quoted value: placeholders and documentation pass', () => { + for (const text of ['your password here', '', 'the password of the admin user']) { + expect(count('a.py', `${HTTP_BASIC}("admin", "${text}")\n`), text).toBe(0); + } + expect(count('a.py', `${HTTP_BASIC}("admin", os.environ["PW"])\n`)).toBe(0); + expect(count('a.py', `${HTTP_BASIC}("admin", "")\n`)).toBe(0); + }); + + it('reports every passphrase-taking command line form, not only the Basic-auth calls', () => { + const lines = [ + ['a.sh', `mysql -u root -p"${phrase}" db`], + ['a.sh', `mysql -u root -p'${phrase}' db`], + ['a.sh', `mysql -u root --password="${phrase}" db`], + ['a.sh', `sshpass -p '${phrase}' ssh host`], + ['a.sh', `rabbitmqadmin -u admin -p "${phrase}" list queues`], + ['a.sh', `rabbitmqadmin --password="${phrase}" list queues`], + ['a.sh', `docker login -u me -p '${phrase}' registry.example.net`], + ['a.sh', `docker login -u me --password "${phrase}" registry.example.net`], + ['a.sh', `redis-cli -a "${phrase}" ping`], + ['a.sh', `curl -u "admin:${phrase}" https://h.example.net`], + ['redis.conf', `requirepass "${phrase}"`], + ['redis.conf', `requirepass '${phrase}'`], + ['redis.conf', `masterauth "${phrase}"`], + ['mosquitto.conf', `password "${phrase}"`], + ['nginx.conf', `proxy_set_header X-Api-Key "${phrase}";`], + ]; + for (const [file, line] of lines) expect(count(file, `${line}\n`), line.replace(phrase, 'P')).toBeGreaterThan(0); + }); + + it('is reported by the tree scan, --range and --history without printing the value', SLOW, () => { + const dir = makeRepo(); + const base = commit(dir, { 'client.py': 'import requests\n' }); + const head = commit(dir, { 'client.py': `import requests\n${HTTP_BASIC}("admin", "${phrase}")\n` }); + for (const args of [[], ['--range', `${base}..${head}`], ['--history']]) { + const found = scan(dir, ...args); + expect(found.status, `${args.join(' ')}: ${found.stderr}`).toBe(1); + expect(found.stderr).toContain('client.py'); + expect(found.stderr).toContain('http-auth-credential'); + expect(`${found.stdout}${found.stderr}`).not.toContain(phrase); + } + }); + + it('scans hostile call text in linear time', SLOW, () => { + const timings = timeInChild(` + const N = 20000; + const texts = [ + 'HTTPBasicAuth("a", "' + 'a\\\\'.repeat(100000), 'HTTPBasicAuth("a", """' + '"'.repeat(100000), "HTTPBasicAuth('a', '''" + "'".repeat(100000), + 'HTTPBasicAuth("a", "b") '.repeat(N), 'auth=("a", \\'' + 'x '.repeat(100000), '.auth(' + '"a",'.repeat(N), 'HTTPBasicAuth(' + 'a'.repeat(200000), + 'x.auth("a", \`' + 'b'.repeat(200000), ('HTTPBasicAuth("' + 'a'.repeat(150) + '", "b\\n').repeat(2000), + ]; + for (const file of ['a.py', 'a.js', 'a.md', 'A.java']) for (const text of texts) { + const started = performance.now(); + scanText(file, text); + timings.push(Math.round(performance.now() - started)); + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + // ------------------------------------------------------------------------- + describe('(2) a merge that completes a credential neither parent held', () => { + const NAME = ['JWT_', 'SECRET'].join(''); + const PEM_HEADER = ['-----BEGIN ', 'ENCRYPTED PRIVATE KEY', '-----'].join(''); + const body = randomString(48, 14002); + const json = (name, value) => `{\n "name": "${name}",\n "kind": "env",\n "value": "${value}"\n}\n`; + const pem = (header, material) => `${header}\nProc-Type: 4,ENCRYPTED\n${material}\n`; + + // Each side branch (a file map) starts at the same commit; the last one checked out merges the others automatically, then a + // later commit puts the base text back, so the pair is gone from the tip. + const mergeAndRemove = (file, base, sides) => { + const dir = makeRepo(); + const start = commit(dir, { [file]: base }); + sides.forEach((files, i) => { + branchFrom(dir, `side${i}`, start); + commit(dir, files); + }); + expect(git(dir, 'checkout', '-q', 'side0').status).toBe(0); + const merge = git(dir, 'merge', '-q', '--no-edit', ...sides.slice(1).map((_, i) => `side${i + 1}`)); + expect(merge.status, merge.stderr).toBe(0); + const merged = git(dir, 'rev-parse', 'HEAD').stdout.trim(); + const head = commit(dir, { [file]: base }); + return { dir, start, merged, head }; + }; + const expectFlagged = ({ dir, start, merged, head }, file) => { + // The pair is gone from the tip, so the tree scan is clean: only the range and the history can see it. + const tip = scan(dir); + expect(tip.status, tip.stderr).toBe(0); + for (const args of [['--range', `${start}..${head}`], ['--history']]) { + const found = scan(dir, ...args); + expect(found.status, `${args.join(' ')}: ${found.stdout}${found.stderr}`).toBe(1); + expect(found.stderr).toContain(merged.slice(0, 7)); + expect(found.stderr).toContain(file); + expect(`${found.stdout}${found.stderr}`).not.toContain(phrase); + expect(`${found.stdout}${found.stderr}`).not.toContain(body); + } + }; + + it.skipIf(!hasGit())('finds a name from one parent and its value from the other (range and history)', SLOW, () => { + const file = 'deploy/env.json'; + const result = mergeAndRemove(file, json('changeme', 'changeme'), [{ [file]: json(NAME, 'changeme') }, { [file]: json('changeme', phrase) }]); + // each parent is clean on its own + expect(count(file, json(NAME, 'changeme'))).toBe(0); + expect(count(file, json('changeme', phrase))).toBe(0); + expectFlagged(result, file); + }); + + it.skipIf(!hasGit())('finds the two halves of a PEM block, header from one parent and key material from the other', SLOW, () => { + const file = 'notes/key.txt'; + const result = mergeAndRemove(file, pem('# key', '# body'), [{ [file]: pem(PEM_HEADER, '# body') }, { [file]: pem('# key', body) }]); + expect(count(file, pem(PEM_HEADER, '# body'))).toBe(0); + expect(count(file, pem('# key', body))).toBe(0); + expectFlagged(result, file); + }); + + it.skipIf(!hasGit())('finds it in an octopus merge', SLOW, () => { + const file = 'deploy/env.json'; + const result = mergeAndRemove(file, json('changeme', 'changeme'), [ + { [file]: json(NAME, 'changeme') }, + { [file]: json('changeme', phrase) }, + { 'docs/other.txt': 'unrelated\n' }, + ]); + expect(git(result.dir, 'show', '-s', '--format=%P', result.merged).stdout.trim().split(' ')).toHaveLength(3); + expectFlagged(result, file); + }); + + it.skipIf(!hasGit())('does not report a credential one parent already held (a base merged into a branch)', SLOW, () => { + const file = 'app.env'; + const lines = (note, token) => `NOTE=${note}\nkeep1=a\nkeep2=b\nkeep3=c\nAPI_${['TOK', 'EN'].join('')}=${token}\n`; + const dir = makeRepo(); + const start = commit(dir, { [file]: lines('one', 'changeme') }); + branchFrom(dir, 'feature', start); + commit(dir, { [file]: lines('two', 'changeme') }); // the branch edits a line far from the credential + expect(git(dir, 'checkout', '-q', 'main').status).toBe(0); + commit(dir, { [file]: lines('one', random) }); // the base gains the credential + const base = git(dir, 'rev-parse', 'HEAD').stdout.trim(); + expect(git(dir, 'checkout', '-q', 'feature').status).toBe(0); + expect(git(dir, 'merge', '-q', '--no-edit', 'main').status).toBe(0); + const head = git(dir, 'rev-parse', 'HEAD').stdout.trim(); + const range = scan(dir, '--range', `${base}..${head}`); + expect(range.status, `${range.stdout}${range.stderr}`).toBe(0); // the base's commit is not the pull request's + expect(scan(dir, '--history').status).toBe(1); // the full audit still blames the base commit + }); + + it.skipIf(!hasGit())('a merge whose result equals a parent adds nothing and a clean merge stays clean', SLOW, () => { + const file = 'deploy/env.json'; + const result = mergeAndRemove(file, json('changeme', 'changeme'), [{ [file]: json('alpha', 'changeme') }, { [file]: json('changeme', 'beta') }]); + for (const args of [[], ['--range', `${result.start}..${result.head}`], ['--history']]) { + const found = scan(result.dir, ...args); + expect(found.status, `${args.join(' ')}: ${found.stdout}${found.stderr}`).toBe(0); + } + }); + }); + + // ------------------------------------------------------------------------- + describe('(3) key material added to a lockfile under a PEM header that did not change', () => { + const PEM_HEADER = ['-----BEGIN ', 'ENCRYPTED PRIVATE KEY', '-----'].join(''); + const material = randomString(48, 14003); + const dek = ['DEK-', 'Info: AES-256-CBC,', randomString(32, 14004, HEX.toUpperCase())].join(''); + const head = '{\n "name": "app",\n "lockfileVersion": 3\n}\n'; + // the header, its fields (as many as the PEM rule reads), a blank line, then the material + const block = (fields, tail) => `${PEM_HEADER}\nProc-Type: 4,ENCRYPTED\n${fields}${dek}\n\n${tail}`; + + it.skipIf(!hasGit()).each(['package-lock.json', 'yarn.lock', 'npm-shrinkwrap.json'])('%s: the material added below an unchanged header is found in the range and the history', SLOW, (file) => { + const dir = makeRepo(); + const header = commit(dir, { [file]: `${head}${block('', '')}` }); + expect(scan(dir).status).toBe(0); // a header alone is not a key + const leaked = commit(dir, { [file]: `${head}${block('', `${material}\n`)}` }); + expect(scan(dir).status).toBe(1); + for (const args of [['--range', `${header}..${leaked}`], ['--history']]) { + const found = scan(dir, ...args); + expect(found.status, `${args.join(' ')}: ${found.stdout}${found.stderr}`).toBe(1); + expect(found.stderr).toContain(leaked.slice(0, 7)); + expect(found.stderr).toContain('private-key-block'); + expect(`${found.stdout}${found.stderr}`).not.toContain(material); + } + }); + + it.skipIf(!hasGit())('finds material added under a header that has several header fields between', SLOW, () => { + const file = 'package-lock.json'; + const fields = ['Comment: a\n', 'Version: 1\n', 'Comment: b\n'].join(''); + const dir = makeRepo(); + const header = commit(dir, { [file]: `${head}${block(fields, '')}` }); + const leaked = commit(dir, { [file]: `${head}${block(fields, `${material}\n`)}` }); + const found = scan(dir, '--range', `${header}..${leaked}`); + expect(found.status, `${found.stdout}${found.stderr}`).toBe(1); + }); + + it.skipIf(!hasGit())('finds a header added above key material that did not change', SLOW, () => { + const file = 'package-lock.json'; + const dir = makeRepo(); + const body = commit(dir, { [file]: `${head}Proc-Type: 4,ENCRYPTED\n${dek}\n\n${material}\n` }); + const leaked = commit(dir, { [file]: `${head}${block('', `${material}\n`)}` }); + expect(scan(dir).status).toBe(1); + const found = scan(dir, '--range', `${body}..${leaked}`); + expect(found.status, `${found.stdout}${found.stderr}`).toBe(1); + }); + + it.skipIf(!hasGit())('a lockfile bump without a PEM header keeps its small context and stays clean', SLOW, () => { + const file = 'package-lock.json'; + const entries = (version) => { + const lines = ['{', ' "packages": {']; + for (let i = 0; i < 3000; i += 1) lines.push(` "node_modules/pkg${i}": { "version": "${i === 1500 ? version : '1.0.0'}" },`); + lines.push(' "node_modules/last": { "version": "1.0.0" }', ' }', '}', ''); + return lines.join('\n'); + }; + const dir = makeRepo(); + const before = commit(dir, { [file]: entries('1.0.0') }); + const after = commit(dir, { [file]: entries('1.0.1') }); + for (const args of [['--range', `${before}..${after}`], ['--history']]) { + const found = scan(dir, ...args); + expect(found.status, `${args.join(' ')}: ${found.stdout}${found.stderr}`).toBe(0); + } + }); + + it.skipIf(!hasGit())('the other lockfile rules read one line, so the two-line context loses nothing for them', SLOW, () => { + // _authToken and registry URLs are single-line lockfile rules: each is found on the added line alone. + const dir = makeRepo(); + const token = randomString(36, 14005); + const before = commit(dir, { 'package-lock.json': '{\n "registry": "https://registry.example.net/"\n}\n' }); + const after = commit(dir, { 'package-lock.json': `{\n "registry": "https://registry.example.net/",\n "_authToken": "${token}"\n}\n` }); + const found = scan(dir, '--range', `${before}..${after}`); + expect(found.status, `${found.stdout}${found.stderr}`).toBe(1); + expect(`${found.stdout}${found.stderr}`).not.toContain(token); + }); + }); +}); + +// --------------------------------------------------------------------------- +// Review round 15: Java properties with a blank separator, whole-value variable references, call-style environment setters. +// Every value is generated or assembled at run time; the names that would look like secrets are split. +// --------------------------------------------------------------------------- +describe('review round 15', () => { + const value = randomString(28, 15001); + const dollarPassword = ['pa$', '$w0rd', 'Zq7!'].join(''); + const JWT = ['JWT_', 'SECRET'].join(''); + const jwtDotted = ['jwt.', 'secret'].join(''); + const DB = ['db_', 'password'].join(''); + const count = (file, text) => scanText(file, text).length; + + const dirs = []; + afterEach(() => { + while (dirs.length > 0) rmSync(dirs.pop(), { recursive: true, force: true }); + }); + const run = (cmd, args, cwd) => spawnSync(cmd, args, { cwd, encoding: 'utf8', timeout: SLOW_TEST_MS, maxBuffer: 64 * 1024 * 1024 }); + const git = (cwd, ...args) => run('git', ['-c', 'user.name=t', '-c', 'user.email=t@example.invalid', '-c', 'commit.gpgsign=false', ...args], cwd); + const commit = (dir, files) => { + for (const [file, content] of Object.entries(files)) { + mkdirSync(path.dirname(path.join(dir, file)), { recursive: true }); + writeFileSync(path.join(dir, file), content); + } + git(dir, 'add', '-A'); + expect(git(dir, 'commit', '-q', '-m', 'c').status).toBe(0); + return git(dir, 'rev-parse', 'HEAD').stdout.trim(); + }; + + // ------------------------------------------------------------------------- + describe('(1) a Java properties line whose key and value are separated by blanks', () => { + const files = ['app.properties', 'application-prod.properties', 'config/db.properties', 'application.properties.local', 'db.properties.bak']; + + it.each(files)('reports keyvalue, keyvalue, key=value and key:value in %s', (file) => { + for (const line of [`${jwtDotted} ${value}`, `${jwtDotted}\t${value}`, ` ${jwtDotted} ${value}`, `${jwtDotted}=${value}`, `${jwtDotted}:${value}`, `${jwtDotted} = ${value}`]) { + expect(count(file, `${line}\n`), line.replace(value, 'V')).toBe(1); + } + }); + + it('reads an escaped blank as part of the key', () => { + expect(count('a.properties', `${['jwt.', 'secret'].join('')}\\ key ${value}\n`)).toBe(1); + expect(count('a.properties', `jwt\\ ${['sec', 'ret'].join('')} ${value}\n`)).toBe(1); + }); + + it('reads a value continued with a trailing backslash', () => { + expect(count('a.properties', `${jwtDotted} \\\n ${value}\n`)).toBe(1); + expect(count('a.properties', `${jwtDotted} ${value.slice(0, 10)}\\\n ${value.slice(10)}\n`)).toBe(1); + expect(count('a.properties', `${jwtDotted}=\\\n ${value}\n`)).toBe(1); + }); + + it('reads a passphrase and a value with a dollar sign', () => { + expect(count('a.properties', `${DB} correct horse battery staple\n`)).toBe(1); + expect(count('a.properties', `${DB} ${dollarPassword}\n`)).toBe(1); + }); + + it('lets placeholders, references, empty values and comments pass', () => { + for (const text of ['${JWT_ID}', '@jwt.secret@', 'changeme', '$JWT_ID', '%(JWT_ID)s', 'your secret here', '']) { + expect(count('a.properties', `${jwtDotted} ${text}\n`), text).toBe(0); + } + expect(count('a.properties', `${jwtDotted}\n`)).toBe(0); + expect(count('a.properties', `${jwtDotted} \n`)).toBe(0); + expect(count('a.properties', `# ${jwtDotted} ${value}\n! ${jwtDotted} ${value}\n`)).toBe(0); + expect(count('a.properties', 'server.port 8080\napp.name Fish Calculator\ntimeout 30\n')).toBe(0); + }); + + it('keeps the message-catalog exemption for prose', () => { + expect(count('messages.properties', `login.password Please type your password here now\n`)).toBe(0); + expect(count('messages_en.properties', `password.reset.help Enter the password you received by mail\n`)).toBe(0); + expect(count('i18n/errors.properties', `${DB}.hint The password of the admin user\n`)).toBe(0); + }); + + it('does not read other file types this way', () => { + expect(count('notes.txt', `${jwtDotted} ${value}\n`)).toBe(0); + expect(count('a.js', `${jwtDotted} ${value}\n`)).toBe(0); + }); + + it('is reported by the tree scan and --range without printing the value', SLOW, () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-r15-')); + dirs.push(dir); + expect(run('git', ['init', '-q', '-b', 'main'], dir).status).toBe(0); + const base = commit(dir, { 'conf/app.properties': 'server.port 8080\n' }); + const head = commit(dir, { 'conf/app.properties': `server.port 8080\n${jwtDotted} ${value}\n` }); + for (const args of [[], ['--range', `${base}..${head}`]]) { + const found = run(process.execPath, [SCANNER, ...args], dir); + expect(found.status, `${args.join(' ')}: ${found.stderr}`).toBe(1); + expect(found.stderr).toContain('app.properties'); + expect(`${found.stdout}${found.stderr}`).not.toContain(value); + } + }); + + it('scans hostile properties text in linear time', SLOW, () => { + const timings = timeInChild(` + const texts = [ + 'a'.repeat(200000), 'a '.repeat(100000), ('k' + ' '.repeat(70)).repeat(3000), '\\\\ '.repeat(100000), 'a\\\\\\n'.repeat(50000), + ('jwt.secret ' + 'x '.repeat(500) + '\\\\\\n').repeat(400), ' '.repeat(200000) + 'a', ('a'.repeat(1100) + ' v\\n').repeat(500), + ]; + for (const file of ['a.properties', 'a.properties.local']) for (const text of texts) { + const started = performance.now(); + scanText(file, text); + timings.push(Math.round(performance.now() - started)); + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + }); + + // ------------------------------------------------------------------------- + describe('(2) a value that is entirely a variable reference is not a hardcoded secret', () => { + const cases = [ + ['a.sh', (n) => `${n}=$password`], + ['a.sh', (n) => `${n}="$password"`], + ['a.env', (n) => `${n}=$password`], + ['a.yml', (n) => `${n}: $password`], + ['a.sh', (n) => `${n}=$dbPassword`], + ['a.sh', (n) => `${n}=$1`], + ['a.sh', (n) => `${n}=\${password}`], + ['a.sh', (n) => `${n}=\${password:-}`], + ['a.sh', (n) => `${n}=$(cat /run/secrets/x)`], + ['a.sh', (n) => `${n}=\`cat /run/secrets/x\``], + ['a.ini', (n) => `${n} = %(DB_PASSWORD)s`], + ['a.ini', (n) => `${n} = %(db_password)d`], + ['a.ini', (n) => `${n} = %%db_pw%%`], + ['a.bat', (n) => `set ${n}=%jwt_pw%`], + ['a.yml', (n) => `${n}: {{ db_pw }}`], + ['a.yml', (n) => `${n}: {{ .Values.db.pw }}`], + ['a.yml', (n) => `${n}: <%= ENV['DB_PW'] %>`], + ['a.yml', (n) => `${n}: #{ENV['DB_PW']}`], + ['a.yml', (n) => `${n}: \${{ secrets.DB_PW }}`], + ['a.properties', (n) => `${n}=@db.pw@`], + ['a.yml', (n) => `${n}: $\${DB_PW}`], + ['docker-compose.yml', (n) => `${n}: \${DB_PW:?err}`], + ['a.pl', (n) => `${n} = $ENV{DB_PW};`], + ['a.php', (n) => `$${n} = $_ENV['DB_PW'];`], + ['a.cmd', (n) => `set ${n}=%ENV%`], + ['a.ps1', (n) => `$${n} = $env:DB_PW`], + ['a.py', (n) => `${n} = os.environ["DB_PW"]`], + ['a.c', (n) => `char* ${n} = getenv("DB_PW");`], + ['a.js', (n) => `const ${n} = process.env.DB_PW;`], + ['A.java', (n) => `String ${n} = System.getenv("DB_PW");`], + ['a.rb', (n) => `${n} = ENV.fetch("DB_PW")`], + ['a.rb', (n) => `${n} = ENV["DB_PW"]`], + ['A.cs', (n) => `var ${n} = Environment.GetEnvironmentVariable("DB_PW");`], + ]; + it.each(cases)('passes a whole-value reference in %s: %s', (file, form) => { + expect(count(file, `${form(DB)}\n`)).toBe(0); + expect(count(file, `${form(JWT)}\n`)).toBe(0); + }); + + it('judges a literal glued to a reference, and a default with a literal', () => { + expect(count('a.sh', `${JWT}=${value}$suffix\n`)).toBe(1); + expect(count('a.sh', `${JWT}=$prefix${value}\n`)).toBe(1); + expect(count('a.sh', `${JWT}=\${password:-${value}}\n`)).toBe(1); + expect(count('a.ini', `${DB} = ${value}%(DB_PASSWORD)s\n`)).toBe(1); + }); + + it('still reports a password that contains or starts with a dollar sign', () => { + for (const file of ['a.sh', 'a.env', 'a.ini', 'a.yml', 'a.properties']) { + expect(count(file, `${DB}=${dollarPassword}\n`), file).toBe(1); + expect(count(file, `${DB}=$${value}\n`), file).toBe(1); + expect(count(file, `${DB}=$${value.toLowerCase()}9\n`), file).toBe(1); + } + expect(count('a.js', `const ${DB} = "${dollarPassword}";\n`)).toBe(1); + }); + + it('exports isPlaceholder with the same verdicts', () => { + for (const text of ['$password', '$dbPassword', '%(DB_PASSWORD)s', '%(x)d', '$ENV{X}', "$_ENV['X']", '$env:X', "<%= ENV['X'] %>", '@x@', '%X%']) { + expect(isPlaceholder(text), text).toBe(true); + } + for (const text of [dollarPassword, `$${value}`, `${value}%(X)s`, `x%(X)`]) expect(isPlaceholder(text), text.slice(0, 6)).toBe(false); + }); + }); + + // ------------------------------------------------------------------------- + describe('(3) a function call that sets an environment variable or property to a literal', () => { + const NAME_PROP = ['jwt.', 'secret'].join(''); + const calls = [ + ['A.cs', (n, v) => `Environment.SetEnvironmentVariable("${n}", "${v}");`], + ['A.cs', (n, v) => `System.Environment.SetEnvironmentVariable("${n}", "${v}");`], + ['A.cs', (n, v) => `Environment.SetEnvironmentVariable("${n}", "${v}", EnvironmentVariableTarget.Machine);`], + ['A.cs', (n, v) => `Environment.SetEnvironmentVariable(@"${n}", @"${v}");`], + ['a.ps1', (n, v) => `[Environment]::SetEnvironmentVariable('${n}', '${v}', 'User')`], + ['a.c', (n, v) => `SetEnvironmentVariableW(L"${n}", L"${v}");`], + ['a.go', (n, v) => `os.Setenv("${n}", "${v}")`], + ['a.go', (n, v) => `os.Setenv("${n}", \`${v}\`)`], + ['a.py', (n, v) => `os.putenv("${n}", "${v}")`], + ['a.py', (n, v) => `os.environ.setdefault("${n}", '${v}')`], + ['a.py', (n, v) => `os.environ["${n}"] = "${v}"`], + ['A.java', (n, v) => `System.setProperty("${NAME_PROP}", "${v}");`], + ['A.java', (n, v) => `props.setProperty("${n}", "${v}");`], + ['A.java', (n, v) => `System.getenv().put("${n}", "${v}");`], + ['A.java', (n, v) => `pb.environment().put("${n}", "${v}");`], + ['a.rb', (n, v) => `ENV["${n}"] = "${v}"`], + ['a.rb', (n, v) => `ENV.store("${n}", "${v}")`], + ['a.c', (n, v) => `putenv("${n}=${v}");`], + ['a.c', (n, v) => `setenv("${n}", "${v}", 1);`], + ['a.c', (n, v) => `_putenv_s("${n}", "${v}");`], + ['a.php', (n, v) => `putenv("${n}=${v}");`], + ['a.php', (n, v) => `apache_setenv('${n}', '${v}');`], + ['a.php', (n, v) => `$_ENV['${n}'] = '${v}';`], + ['a.ts', (n, v) => `Deno.env.set("${n}", "${v}");`], + ['a.js', (n, v) => `process.env["${n}"] = "${v}";`], + ['a.rs', (n, v) => `std::env::set_var("${n}", "${v}");`], + ['a.ex', (n, v) => `System.put_env("${n}", "${v}")`], + ['a.pl', (n, v) => `$ENV{${n}} = '${v}';`], + ['a.pl', (n, v) => `$config{'${n}'} = "${v}";`], + ['a.cmd', (n, v) => `setx ${n} ${v}`], + ]; + it.each(calls)('reports a literal set in %s: %s', (file, form) => { + expect(count(file, `${form(JWT, value)}\n`)).toBe(1); + }); + + it('reports a passphrase and a fenced block in Markdown', () => { + expect(count('A.cs', `Environment.SetEnvironmentVariable("${JWT}", "correct horse battery staple");\n`)).toBe(1); + expect(count('a.md', `\`\`\`csharp\nEnvironment.SetEnvironmentVariable("${JWT}", "${value}");\n\`\`\`\n`)).toBe(1); + expect(count('a.md', `Call Environment.SetEnvironmentVariable("${JWT}", "${value}") in your code.\n`)).toBe(0); + }); + + it('passes placeholders, references, variables and names that are not secrets', () => { + const same = [ + (v) => `Environment.SetEnvironmentVariable("${JWT}", "${v}");`, + (v) => `os.Setenv("${JWT}", "${v}")`, + (v) => `putenv("${JWT}=${v}");`, + (v) => `setenv("${JWT}", "${v}", 1);`, + (v) => `System.setProperty("${NAME_PROP}", "${v}");`, + ]; + for (const form of same) { + for (const v of ['changeme', '${JWT_ID}', '$OTHER', '%(X)s', '', '']) expect(count('x.cs', `${form(v)}\n`), form('V') + v).toBe(0); + expect(count('x.cs', `${form(value)}`.replace(JWT, 'PATH').replace(NAME_PROP, 'user.dir')), form('V')).toBe(0); + } + expect(count('A.cs', `Environment.SetEnvironmentVariable("${JWT}", Configuration["X"]);\n`)).toBe(0); + expect(count('A.cs', `Environment.SetEnvironmentVariable("${JWT}", $"{prefix}${value}");\n`)).toBe(0); + expect(count('a.go', `os.Setenv("${JWT}", token)\n`)).toBe(0); + expect(count('a.c', `setenv("${JWT}", getenv("OTHER"), 1);\n`)).toBe(0); + expect(count('a.py', `os.environ["${JWT}"] = other\n`)).toBe(0); + expect(count('a.pl', `$ENV{${JWT}} = $other;\n`)).toBe(0); + }); + + it('scans hostile call text in linear time', SLOW, () => { + const timings = timeInChild(` + const N = 20000; + const J = 'JWT_' + 'SECRET'; + const env = 'Environment.' + 'SetEnvironmentVariable('; + const texts = [ + env + '"' + J + '", "' + 'a\\\\'.repeat(100000), 'set' + 'env(' + '"'.repeat(100000), 'put' + 'env("' + J + '=' + 'x'.repeat(200000), + ('os.' + 'Setenv("' + J + '", "b") ').repeat(N), 'x.set' + 'Property(' + 'a.'.repeat(100000), 'a'.repeat(100000) + '.set' + 'Property("' + J + '", "b")', + '$ENV{' + ' '.repeat(200000), ('$c{"' + J + '"} =' + ' '.repeat(20)).repeat(5000), env + '@$LuU'.repeat(40000), + env + '"' + J + '", """' + 'a'.repeat(200000), + ]; + for (const file of ['a.cs', 'a.c', 'a.md', 'A.java']) for (const text of texts) { + const started = performance.now(); + scanText(file, text); + timings.push(Math.round(performance.now() - started)); + }`); + for (const ms of timings) expect(ms).toBeLessThan(HOSTILE_LIMIT_MS); + }); + + it('is reported by the tree scan and --range without printing the value', SLOW, () => { + const dir = mkdtempSync(path.join(tmpdir(), 'check-secrets-r15-')); + dirs.push(dir); + expect(run('git', ['init', '-q', '-b', 'main'], dir).status).toBe(0); + const base = commit(dir, { 'src/Program.cs': 'class P { }\n' }); + const head = commit(dir, { 'src/Program.cs': `class P { void M() { System.Environment.SetEnvironmentVariable("${JWT}", "${value}"); } }\n` }); + for (const args of [[], ['--range', `${base}..${head}`]]) { + const found = run(process.execPath, [SCANNER, ...args], dir); + expect(found.status, `${args.join(' ')}: ${found.stderr}`).toBe(1); + expect(found.stderr).toContain('Program.cs'); + expect(`${found.stdout}${found.stderr}`).not.toContain(value); + } + }); + }); +}); diff --git a/docs/ENVIRONMENT_VARIABLES.md b/docs/ENVIRONMENT_VARIABLES.md index b268d01..d05c97f 100644 --- a/docs/ENVIRONMENT_VARIABLES.md +++ b/docs/ENVIRONMENT_VARIABLES.md @@ -74,10 +74,11 @@ If protected endpoints return 401 after Firebase sign-in, check: ## Local Development Setup -1. Copy `.env.example` to `.env.development`: +1. Create an untracked local file from the template. Vite loads `app/.env.development.local` after `app/.env.development`, so its values win, and `.gitignore` keeps it out of git: ```bash - cp app/.env.example app/.env.development + [ -e app/.env.development.local ] || cp app/.env.example app/.env.development.local ``` + Do **not** copy the template over `app/.env.development` or `app/.env.production`. Those two files are tracked in git. 2. Fill in your values **without quotes**: ```bash @@ -87,10 +88,10 @@ If protected endpoints return 401 after Firebase sign-in, check: VITE_FIREBASE_AUTH_DOMAIN=your-project.firebaseapp.com VITE_FIREBASE_APP_ID=your-firebase-web-app-id VITE_FIREBASE_MESSAGING_SENDER_ID=your-sender-id - DATABASE_URL=your-connection-string-here ``` + The frontend never reads `DATABASE_URL`, so it does not belong in this file. For `vercel dev`, run `vercel pull`, which stores the project settings and environment variables under `.vercel/`. For the scripts in `scripts/`, `export DATABASE_URL=...` in your shell. See "For new developers" in [`SECURITY_NOTICE.md`](../SECURITY_NOTICE.md). -3. Never commit `.env`, `.env.development`, or `.env.production` files - they're gitignored for security. +3. Never put a real value in a tracked file. `.gitignore` covers only `.env`, `.env.local` and the `.env*.local` files (such as `app/.env.development.local`). `app/.env.development`, `app/.env.production`, `app/.env.example` and `server/.env.example` are **tracked** and must stay placeholder-only; the `Secret scan` CI job fails if a real-looking credential lands in a tracked file. If one already did, follow [`SECURITY_NOTICE.md`](../SECURITY_NOTICE.md). 4. For the local Express server (`server/server.js`), create `server/.env` (or `.env.local`) with `FIREBASE_PROJECT_ID`, `JWT_SECRET`, and `ALLOWED_ORIGINS` that match your dev URLs. See `server/.env.example` for defaults. diff --git a/scripts/check-secrets.mjs b/scripts/check-secrets.mjs new file mode 100644 index 0000000..2e0cb85 --- /dev/null +++ b/scripts/check-secrets.mjs @@ -0,0 +1,5380 @@ +#!/usr/bin/env node +/** + * check-secrets.mjs - dependency-free secret scanner for git-tracked files. + * + * Usage (from anywhere inside the repository): + * node scripts/check-secrets.mjs scan every git-tracked text file (this is what CI runs) + * node scripts/check-secrets.mjs --history scan the ADDED lines of every commit on every ref + * node scripts/check-secrets.mjs --range .. scan the ADDED lines of the commits in that range (CI) + * node scripts/check-secrets.mjs --help + * + * Exit codes: 0 = clean, 1 = potential secret found (or a tracked file that could not be scanned: + * text over the size limit, unreadable), 2 = usage or git error, or a --history / --range audit that did + * not look at everything (version over the size limit, shallow clone, a commit that is not present). + * + * Fail closed: the only content skipped on purpose is gitlinks (submodule + * pointers) and verified-binary files (a NUL byte in the first 8 KB AND a known binary signature such as PNG, + * ZIP or PDF; a stray NUL alone never exempts a file), and the OK line counts them. Everything else that cannot + * be examined makes the run fail with a message; nothing is skipped quietly. + * + * Redaction guarantee: findings carry only { path, line, rule }. The matched text is + * never stored, printed or logged, not even partially. Keep it that way. + * + * Placeholders: values such as your_..., change-me, user:password@host, example, + * xxxx, <...>, ..., REDACTED, empty values and process.env / import.meta.env + * references are ignored (see isPlaceholder). To silence a single false positive, + * put `check-secrets:allow` in a comment on the same line (any line of a multi-line + * match works, and so does the value line of a name/value pair split across lines); + * it stays visible in review. + * + * How name = value assignments are judged (rule `secret-assignment`): + * - The NAME is split into words (JWT_SECRET, jwtSecret, jwt-secret). A name that ENDS in + * a secret noun (secret, password, pass, pwd, token, api/private/signing/... key) is + * "strong"; a name that merely contains one (TOKEN_ENDPOINT) is "weak". + * - Env, config and Markdown files: a strong name with any non-placeholder value of 8+ + * characters is a finding (no entropy test, so short passwords and passphrases count). + * A weak name needs a random-looking value. + * - Everything else (source code, SQL, HTML, ...): only a QUOTED literal that looks random, + * so ordinary identifiers and test fixtures do not trip it. + * - A value with spaces is exempt only as text: a sentence (looksLikeSentence) inside a message catalog path, or + * documentation about the credential (looksLikeDocumentation) anywhere; a passphrase that merely contains common words + * ("correct horse battery and staple", "This is the way.") is a finding. + * - A URL value is judged only under a strong name, and only for credential-looking parts (signed-URL parameters, + * webhook path tokens, a token as the user name). Webhook URLs are also a rule of their own (webhook-url). + * - Native credential files (.netrc, .pgpass, .git-credentials, .npmrc, .pypirc, .aws/credentials, docker config, + * kubeconfig, .htpasswd, .my.cnf, .s3cfg, Terraform, .curlrc, .vault-token) have path-specific matchers + * (rule credential-file) and a 4-character value minimum; netrc, pgpass and docker auth are also recognised by + * content in any file (notes, scripts that write them). + * - Shell syntaxes with no "=" are read too: fish `set -gx NAME value ...`, csh `setenv NAME value`, Windows `setx` and + * `set "NAME=value"`, `export NAME value`, PowerShell `$env:NAME = 'v'` / SetEnvironmentVariable / Set-Item Env:NAME, fish_variables + * SETUVAR lines, in shell scripts, config files, heredocs written into them and fenced Markdown blocks. + * - Literals over several lines are judged like a quoted value: TOML/Python triple quotes, HCL/Ruby/Perl/PHP heredocs, PowerShell + * here-strings, template literals, quotes closed on a later line, backslash / INI continuation lines, YAML scalars on the next line. + * They are read up to 200 lines / 32 KB; a literal that does not end within that is reported (fail closed). + * - Credentials that are not `name = value`: HTTP `Authorization` values with a scheme (http-auth-credential; Basic is decoded and + * judged by its password), Terraform `variable "api_token" { default = ... }` labels, and whitespace-delimited service directives + * keyed on their own names (config-directive-secret: Redis requirepass / ACL, HAProxy userlist, Mosquitto, nginx / Apache header + * and SetEnv directives, ...). Only literals count; references, placeholders and paths pass. + * - Name and value fields in one JSON/YAML/HCL/XML object or call are matched in either order (secret-name-value-pair), + * and secrets passed on a command line (gh secret set, vercel env add, aws ssm put-parameter, ...) are their own rule. + * + * The index blob AND the working-tree file are both scanned whenever they differ (compared by git blob id). + * --history and --range skip a file version that is verified binary (the same test as above, on the version's own first 8 KB) and + * count it as skipped, so a large image never fails a run as "oversize"; a text version over the size limit still exits 2. + * + * --history is for the repository owner to run locally. CI does NOT run it: the known + * leak from issue #22 lives in history forever and would fail every build. A shallow + * clone only has part of the history, so an incomplete audit (shallow, or any version not + * scanned) exits 2 and never prints "no hits". + */ + +import { spawn, spawnSync } from 'node:child_process'; +import { Buffer } from 'node:buffer'; +import { closeSync, lstatSync, openSync, readFileSync, readlinkSync, readSync, realpathSync } from 'node:fs'; +import path from 'node:path'; +import process from 'node:process'; +import { StringDecoder } from 'node:string_decoder'; +import { fileURLToPath } from 'node:url'; + +// --------------------------------------------------------------------------- +// Skip list and file decoding +// --------------------------------------------------------------------------- + +const ALLOW_MARKER = 'check-secrets:allow'; +const MAX_FILE_BYTES = 5 * 1024 * 1024; +const SNIFF_BYTES = 8000; + +// Lockfiles are full of integrity hashes that look like high-entropy secrets, so the generic entropy and secret-name +// rules do not run on them. They are NOT skipped: a dependency URL can carry a credential (https://user:@registry/...), +// and a lockfile can hold an auth field. Only rules flagged `lockfile: true` (URL passwords, private keys, provider-shaped +// tokens, webhook URLs) and the targeted `lockfile-credential` rule run on them, after ordinary digests are blanked +// (sanitizeLockfile). Exact names only: other *.lock files are scanned with every rule. +const LOCKFILE_NAMES = new Set([ + 'package-lock.json', + 'npm-shrinkwrap.json', + 'yarn.lock', + 'pnpm-lock.yaml', + 'bun.lockb', + 'bun.lock', + 'composer.lock', + 'gemfile.lock', + 'cargo.lock', + 'poetry.lock', + 'pipfile.lock', + 'go.sum', +]); + +/** True for lockfiles (exact base names). They are scanned with the lockfile rules only, not skipped. */ +export function isLockfile(filePath) { + const base = path.posix.basename(filePath.split(path.sep).join('/')).toLowerCase(); + return LOCKFILE_NAMES.has(base); +} + +// A lockfile is machine-generated and can be far larger than a source file; it gets a higher (still fail-closed) limit. +const MAX_LOCKFILE_BYTES = 16 * 1024 * 1024; +const sizeLimit = (filePath) => (isLockfile(filePath) ? MAX_LOCKFILE_BYTES : MAX_FILE_BYTES); + +// Ordinary digests, blanked (same length, so line numbers stay) before the lockfile rules run. Every quantifier is a +// fixed count or a bounded class run, so this is linear. Only digests of the exact size of their algorithm are blanked: +// a longer or shorter string after "sha512-" is not an integrity value and stays visible to the token rules. +const LOCKFILE_DIGESTS = [ + /sha512-[A-Za-z0-9+/]{86}={0,2}/g, + /sha384-[A-Za-z0-9+/]{64}={0,2}/g, + /sha256-[A-Za-z0-9+/]{43}={0,2}/g, + /sha1-[A-Za-z0-9+/]{27}={0,2}/g, + /\bh1:[A-Za-z0-9+/]{43}=/g, // go.sum + /(? ' '.repeat(m.length)); + return out.replace(LOCKFILE_HEX_DIGEST, (m, lead, hex) => lead + ' '.repeat(hex.length)); +} + +// Leading bytes of common binary formats. A file is "verified binary" only when it has a NUL byte in its first +// 8 KB AND starts with one of these; a NUL alone proves nothing (a stray NUL must not exempt an .env file). +const BINARY_SIGNATURES = [ + [0x89, 0x50, 0x4e, 0x47], // PNG + [0xff, 0xd8, 0xff], // JPEG + [0x47, 0x49, 0x46, 0x38], // GIF + [0x50, 0x4b, 0x03, 0x04], // ZIP, jar, docx, xlsx, ... + [0x50, 0x4b, 0x05, 0x06], + [0x25, 0x50, 0x44, 0x46, 0x2d], // %PDF- + [0x7f, 0x45, 0x4c, 0x46], // ELF + [0xca, 0xfe, 0xba, 0xbe], // Mach-O fat / Java class + [0xcf, 0xfa, 0xed, 0xfe], // Mach-O + [0x00, 0x61, 0x73, 0x6d], // WebAssembly + [0x1f, 0x8b, 0x08], // gzip + [0x42, 0x5a, 0x68, 0x39, 0x31, 0x41, 0x59], // bzip2 (level 9 block header) + [0xfd, 0x37, 0x7a, 0x58, 0x5a, 0x00], // xz + [0x37, 0x7a, 0xbc, 0xaf, 0x27, 0x1c], // 7z + [0x28, 0xb5, 0x2f, 0xfd], // zstd + [0x52, 0x61, 0x72, 0x21], // Rar! + [0x77, 0x4f, 0x46, 0x46], // wOFF + [0x77, 0x4f, 0x46, 0x32], // wOF2 + [0x00, 0x01, 0x00, 0x00], // TrueType + [0x4f, 0x54, 0x54, 0x4f], // OpenType + [0x00, 0x00, 0x01, 0x00], // ico + [0x49, 0x49, 0x2a, 0x00], // TIFF + [0x4d, 0x4d, 0x00, 0x2a], + [0x52, 0x49, 0x46, 0x46], // RIFF (webp, wav, avi) + [0x4f, 0x67, 0x67, 0x53], // Ogg + [0x53, 0x51, 0x4c, 0x69, 0x74, 0x65, 0x20, 0x66], // SQLite +].map((bytes) => Buffer.from(bytes)); + +const hasSignature = (head) => + BINARY_SIGNATURES.some((sig) => head.length >= sig.length && head.subarray(0, sig.length).equals(sig)) || + (head.length >= 12 && head[0] === 0 && head[1] === 0 && head.subarray(4, 8).toString('latin1') === 'ftyp'); // mp4, heic, avif + +// BOM-less UTF-16 of mostly ASCII text has a NUL in every other byte and none in the rest. +function guessBomlessUtf16(head) { + const pairs = head.length >> 1; + if (pairs < 4) return null; + let evenNuls = 0; + let oddNuls = 0; + for (let i = 0; i + 1 < head.length; i += 2) { + if (head[i] === 0) evenNuls += 1; + if (head[i + 1] === 0) oddNuls += 1; + } + if (oddNuls >= pairs * 0.3 && evenNuls <= pairs * 0.02) return 'le'; + if (evenNuls >= pairs * 0.3 && oddNuls <= pairs * 0.02) return 'be'; + return null; +} + +// BOM-less UTF-32 of mostly ASCII text: three NULs in every four bytes (the last three for LE, the first three for BE). +function guessBomlessUtf32(head) { + const quads = head.length >> 2; + if (quads < 4) return null; + let le = 0; + let be = 0; + for (let i = 0; i + 3 < head.length; i += 4) { + if (head[i] !== 0 && head[i + 1] === 0 && head[i + 2] === 0 && head[i + 3] === 0) le += 1; + if (head[i] === 0 && head[i + 1] === 0 && head[i + 2] === 0 && head[i + 3] !== 0) be += 1; + } + if (le >= quads * 0.7) return 'le'; + if (be >= quads * 0.7) return 'be'; + return null; +} + +const swapped = (bytes) => Buffer.from(bytes.subarray(0, bytes.length & ~1)).swap16(); + +function decodeUtf32(bytes, littleEndian) { + const parts = []; + let chunk = []; + for (let i = 0; i + 3 < bytes.length; i += 4) { + const cp = littleEndian ? bytes.readUInt32LE(i) : bytes.readUInt32BE(i); + chunk.push(cp <= 0x10ffff ? cp : 0xfffd); + if (chunk.length === 4096) { + parts.push(String.fromCodePoint(...chunk)); + chunk = []; + } + } + parts.push(String.fromCodePoint(...chunk)); + return parts.join(''); +} + +/** Which multi-byte Unicode encoding `buffer` is in, by BOM first and then by the NUL pattern of its first 8 KB. */ +function detectWideEncoding(buffer) { + if (buffer.length >= 4 && buffer[0] === 0xff && buffer[1] === 0xfe && buffer[2] === 0 && buffer[3] === 0) return 'utf32le-bom'; + if (buffer.length >= 4 && buffer[0] === 0 && buffer[1] === 0 && buffer[2] === 0xfe && buffer[3] === 0xff) return 'utf32be-bom'; + if (buffer.length >= 2 && buffer[0] === 0xff && buffer[1] === 0xfe) return 'utf16le-bom'; + if (buffer.length >= 2 && buffer[0] === 0xfe && buffer[1] === 0xff) return 'utf16be-bom'; + const head = buffer.subarray(0, SNIFF_BYTES); + if (!head.includes(0)) return null; + const wide32 = guessBomlessUtf32(head); + if (wide32) return `utf32${wide32}`; + const wide16 = guessBomlessUtf16(head); + return wide16 ? `utf16${wide16}` : null; +} + +/** + * True only for content that is positively binary: a NUL byte in the first 8 KB, not a UTF-16/32 text file, and a + * known binary signature at the start. `head` is the first bytes of the file (8 KB is enough). + */ +export function isBinaryContent(head) { + const sniff = head.subarray(0, SNIFF_BYTES); + return sniff.includes(0) && detectWideEncoding(sniff) === null && hasSignature(sniff); +} + +/** + * Decode file bytes to text. UTF-8 (with or without BOM), UTF-16 and UTF-32 (with or without BOM) are decoded. + * Any other content, including text with a stray NUL byte, is decoded as UTF-8 with the NULs removed and scanned. + * Returns null only for verified-binary content (see isBinaryContent). A secret is ASCII, so the invalid-UTF-8 + * replacement of non-UTF-8 text encodings does not hide one. + */ +export function decodeText(buffer) { + if (isBinaryContent(buffer)) return null; + const wide = detectWideEncoding(buffer); + let text; + if (wide === 'utf32le-bom') text = decodeUtf32(buffer.subarray(4), true); + else if (wide === 'utf32be-bom') text = decodeUtf32(buffer.subarray(4), false); + else if (wide === 'utf32le') text = decodeUtf32(buffer, true); + else if (wide === 'utf32be') text = decodeUtf32(buffer, false); + else if (wide === 'utf16le-bom') text = buffer.subarray(2, 2 + ((buffer.length - 2) & ~1)).toString('utf16le'); + else if (wide === 'utf16be-bom') text = swapped(buffer.subarray(2)).toString('utf16le'); + else if (wide === 'utf16le') text = buffer.subarray(0, buffer.length & ~1).toString('utf16le'); + else if (wide === 'utf16be') text = swapped(buffer).toString('utf16le'); + else { + const hasUtf8Bom = buffer.length >= 3 && buffer[0] === 0xef && buffer[1] === 0xbb && buffer[2] === 0xbf; + text = buffer.toString('utf8', hasUtf8Bom ? 3 : 0); + } + // A NUL that is left over (a stray byte, or a mixed-encoding file) must not split a name or a value. + return text.includes('\u0000') ? text.replace(/\u0000/g, '') : text; +} + +// --------------------------------------------------------------------------- +// Names +// --------------------------------------------------------------------------- + +/** Lower-case words of an identifier: JWT_SECRET, jwtSecret, jwt-secret, jwt.secret all give jwt, secret. */ +function nameWords(name) { + return String(name) + .replace(/([a-z0-9])([A-Z])/g, '$1 $2') + .replace(/([A-Z])(?=[A-Z][a-z])/g, '$1 ') + .split(/[^A-Za-z0-9]+/) + .filter(Boolean) + .map((w) => w.toLowerCase()); +} + +// A name whose LAST word is one of these holds a secret. +const STRONG_LAST_WORDS = new Set([ + 'secret', 'password', 'passwd', 'pwd', 'pass', 'passphrase', 'token', +]); +// ...as does + "key" (apiKey, PRIVATE_KEY, signing_key, ...). +const KEY_QUALIFIERS = new Set([ + 'api', 'private', 'secret', 'access', 'signing', 'encryption', 'auth', 'master', 'session', 'hmac', 'role', +]); +const STRONG_MERGED_SUFFIX = + /(?:secret|password|passwd|pwd|token|apikey|privatekey|secretkey|accesskey|signingkey|encryptionkey|authkey|masterkey|sessionkey|hmackey)$/; +// Words that make a name "weak": it mentions a secret without ending in one (TOKEN_ENDPOINT, PASSWORD_HINT). +const WEAK_WORDS = new Set([ + 'secret', 'secrets', 'password', 'passwords', 'passwd', 'pwd', 'pass', 'passphrase', 'token', 'tokens', + 'credential', 'credentials', 'salt', 'pepper', +]); +const WEAK_MERGED_SUBSTRING = /secret|passw(?:or)?d|token|privatekey|apikey|credential/; +// Credential variables whose names do not end in a secret noun: sshpass reads SSHPASS, redis-cli reads REDISCLI_AUTH, Dovecot's LDAP bind is dnpass. +const EXACT_STRONG_MERGED = new Set(['sshpass', 'rediscliauth', 'dnpass']); + +/** @returns {'strong' | 'weak' | null} how secret-like an identifier is (see the file header) */ +export function secretNameKind(name) { + const words = nameWords(name); + while (words.length > 1 && /^\d+$/.test(words[words.length - 1])) words.pop(); + if (words.length === 0) return null; + const bare = words.map((w) => w.replace(/\d+$/, '')); + const last = bare[bare.length - 1]; + const merged = bare.join(''); + if (STRONG_LAST_WORDS.has(last) || STRONG_MERGED_SUFFIX.test(merged) || EXACT_STRONG_MERGED.has(merged)) return 'strong'; + if (last === 'key' && KEY_QUALIFIERS.has(bare[bare.length - 2])) return 'strong'; + // A bare "_KEY" (SENDGRID_KEY, STRIPE_KEY) may be a credential or a cache key: only a random-looking value counts. + if (last === 'key' && bare.length > 1) return 'weak'; + if (bare.some((w) => WEAK_WORDS.has(w)) || WEAK_MERGED_SUBSTRING.test(merged)) return 'weak'; + for (let i = 0; i + 1 < bare.length; i += 1) { + if (bare[i + 1] === 'key' && KEY_QUALIFIERS.has(bare[i])) return 'weak'; + } + return null; +} + +// --------------------------------------------------------------------------- +// Placeholder detection +// --------------------------------------------------------------------------- + +// Spans that stand in for the real value: , [YOUR-KEY], ..., ***, ___, xxx. +const MARKER_SPANS = /<[^<>\n]{1,80}>|\[[A-Za-z][A-Za-z _-]{0,60}\]|\.{3,}|…|\*{3,}|_{3,}|x{3,}|X{3,}/g; + +// Template and environment references. Interpolation syntax is unambiguous, so it counts anywhere; +// a bare $NAME counts only when the WHOLE value is that shape, so `$` + random letters does not. +const INTERPOLATION = /\$\{|\{\{|#\{|^\$\(|^`/; +const WHOLE_REFERENCE = new RegExp( + '^(?:' + + [ + String.raw`\$[A-Z_][A-Z0-9_]*`, // $NAME + String.raw`\$[a-z][a-z0-9]*_[a-z0-9_]*`, // $db_password + // $password, $dbPassword, $DbPassword: one word, or a few capitalised words, and at most two digits. `$` followed by a longer + // or mixed run (letters and digits together, or a long run of capitals inside) reads as a password that starts with `$`. + String.raw`\$[A-Za-z][a-z]{0,40}(?:[A-Z][a-z]{1,20}){0,4}[0-9]{0,2}`, + String.raw`\$[0-9]{1,2}`, // $1 (a positional parameter) + String.raw`\$[Ee][Nn][Vv]:[A-Za-z_][A-Za-z0-9_]{0,63}`, // PowerShell $env:NAME + String.raw`\$ENV\{[A-Za-z_][A-Za-z0-9_]{0,63}\}`, // Perl $ENV{NAME} + String.raw`\$_(?:ENV|SERVER)\[["']?[A-Za-z_][A-Za-z0-9_]{0,63}["']?\]`, // PHP $_ENV['NAME'] + String.raw`%\([A-Za-z_][A-Za-z0-9_.:-]{0,64}\)[sdifr]`, // Python / configparser %(name)s + String.raw`%[A-Za-z_][A-Za-z0-9_]*%`, // cmd %NAME% + String.raw`%%[A-Za-z_][A-Za-z0-9_.-]*%%`, // Ant / Maven build tokens + String.raw`@[A-Za-z_][A-Za-z0-9_.-]*@`, // autoconf / Maven filtering @name@ + String.raw`<%[=-]?[^%\n]{1,200}%>`, // ERB / EJS <%= ENV['X'] %> + String.raw`[@?]\+?(?:[a-z]+:)?[a-z]+\/[A-Za-z0-9_.]+`, // Android @string/name, ?attr/name + ].join('|') + + ')$', +); +// (%%NAME%% and @NAME@ are build-time substitution tokens (Ant, Maven filtering, autoconf); @string/name and ?attr/name are Android resource references.) +const CODE_REFERENCE = /process\.env|import\.meta\.env|os\.environ|\bgetenv|\bENV\[|\bENV\.(?:fetch|dig)\b|\bDeno\.env\b|GetEnvironmentVariable|\bgetProperty\(/; +// Encrypted or hashed forms are not plaintext credentials. +const NON_SECRET_FORMS = /^(?:ENC\[|\$ANSIBLE_VAULT|\$2[abxy]?\$\d{2}\$|\$argon2|\$pbkdf2|\$scrypt|\$apr1\$|\{SHA\})/; +// AWS documentation keys end in EXAMPLE / EXAMPLEKEY. +const EXAMPLE_SUFFIX = /EXAMPLE(?:KEY)?$/; +// "Bearer eyJhbGciOi..." A trailing ellipsis means the author cut the value off, so it cannot be a working credential. +const TRUNCATED = /(?:\.{3,}|…)$/; +// Three or more plain words are a passphrase with a trailing ellipsis, not a cut-off token: "correct horse battery..." is +// judged as a phrase like the same words without the dots. +const PLAIN_PHRASE_WORD = /^\p{L}[\p{L}'’-]*$/u; +function isTruncated(value) { + if (!TRUNCATED.test(value)) return false; + const pieces = value.replace(TRUNCATED, '').trim().split(/\s+/); + return !(pieces.length >= 3 && pieces.every((piece) => PLAIN_PHRASE_WORD.test(piece))); +} + +// A letter run equal to one of these makes the value a placeholder wherever it sits. +const VERY_STRONG_RUNS = new Set([ + 'example', 'examples', 'dummy', 'placeholder', 'redacted', 'changeme', 'replaceme', +]); +// Separator- or camelCase-delimited words that make the value a placeholder ("your_key", "fake_token_1"). +// Deliberately NOT here: change, enter, here, todo, foo, bar, none. Those only count in the phrases +// below or when the whole value is made of label words, so Change2024! or Spring_2024_todo stay real. +const SEGMENT_WORDS = new Set(['your', 'yours', 'sample', 'fake', 'mock']); +const PLACEHOLDER_PHRASES = + /(?:^|[^a-z])(?:change|replace)[-_ ]?(?:me|this|it|with)(?:[^a-z]|$)|(?:^|[^a-z])(?:enter|insert|paste|put|add|set|type)[-_ ](?:your|the|a|an|my)(?:[^a-z]|$)|[-_ ]here$/; +// A value made ONLY of these words (plus separators and pure numbers) is a label, e.g. "password", "new_secret_key". +const LABEL_WORDS = new Set([ + 'password', 'passwd', 'pass', 'pwd', 'secret', 'key', 'token', 'user', 'username', 'host', 'hostname', + 'db', 'dbname', 'database', 'value', 'string', 'name', 'id', 'api', 'jwt', 'apikey', 'credentials', + 'required', 'optional', 'hidden', 'masked', 'empty', 'unset', 'boolean', 'number', 'base64', + 'long', 'random', 'generated', 'enter', 'insert', 'paste', 'here', 'change', 'replace', 'todo', 'tbd', + 'fixme', 'foo', 'bar', 'baz', 'none', 'null', 'undefined', 'your', 'yours', 'sample', 'fake', 'mock', + 'a', 'an', 'the', 'of', 'to', 'for', 'in', 'is', 'this', 'that', 'with', 'and', 'my', 'new', 'old', 'me', 'it', +]); + +function isMostlyMarkers(value) { + let covered = 0; + for (const m of value.matchAll(MARKER_SPANS)) covered += m[0].length; + if (covered === 0) return false; + // A marker only counts when it stands for most of the value ("AIza...", "xxxxxxxx", ""), + // not when it trails a long real-looking string. + return value.length - covered <= 8 || covered * 2 >= value.length; +} + +function isPlaceholderWords(value) { + if (EXAMPLE_SUFFIX.test(value)) return true; + const lower = value.toLowerCase(); + if (PLACEHOLDER_PHRASES.test(lower)) return true; + if (lower.split(/[^a-z]+/).some((run) => VERY_STRONG_RUNS.has(run))) return true; + const words = nameWords(value); + if (words.some((w) => SEGMENT_WORDS.has(w))) return true; + return ( + /^[A-Za-z0-9 _.-]+$/.test(value) && + words.some((w) => !/^\d+$/.test(w)) && + words.every((w) => /^\d+$/.test(w) || LABEL_WORDS.has(w)) + ); +} + +/** True when `raw` is empty, a template reference, or an obvious dummy value. */ +export function isPlaceholder(raw) { + const value = String(raw ?? '') + .trim() + .replace(/^["'`]+|["'`]+$/g, '') + .trim(); + if (value === '' || !/[A-Za-z0-9]/.test(value)) return true; + if (INTERPOLATION.test(value) || WHOLE_REFERENCE.test(value) || CODE_REFERENCE.test(value)) return true; + if (NON_SECRET_FORMS.test(value) || isTruncated(value) || isMostlyMarkers(value)) return true; + return isPlaceholderWords(value); +} + +// Exact sample values that appear in documentation, scoped to the file that shows them. +// Keep this tiny: each entry silences one string in one file, never a pattern or a whole file. +// The sample must be the ENTIRE assigned value (optionally quoted): "password": "" is exempt, +// PASSWORD=!more or PASSWORD=prefix- is not. +const PATH_SAMPLES = [ + // Sample registration/login request body for the example user "fisherman_joe". + { path: 'docs/API.md', value: 'securePassword123' }, +].map((sample) => ({ + ...sample, + pattern: new RegExp( + String.raw`^[\x22'\x60]?[\w$.-]{1,81}[\x22'\x60]?[ \t]*(?:=>|:=|=|:)[ \t]*[\x22'\x60]?${sample.value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}[\x22'\x60]?$`, + ), +})); + +const isPathSample = (filePath, matchedText) => + PATH_SAMPLES.some((sample) => sample.path === filePath && sample.pattern.test(matchedText)); + +// Only reserved documentation hosts and loopback may hide a password. A host that merely +// contains the word "example" (an RDS or Neon endpoint named after the app) may not. +function isDocumentationHost(rawHost) { + const host = rawHost.toLowerCase().replace(/:\d*$/, '').replace(/^\[|\]$/g, ''); + return ( + host === 'localhost' || + host.endsWith('.localhost') || + host === '127.0.0.1' || + host === '::1' || + host === '0.0.0.0' || + /(?:^|\.)example(?:\.(?:com|org|net))?$/.test(host) || + host.endsWith('.test') || + host.endsWith('.invalid') + ); +} + +// --------------------------------------------------------------------------- +// Randomness +// --------------------------------------------------------------------------- + +function shannonEntropy(text) { + const counts = new Map(); + for (const ch of text) counts.set(ch, (counts.get(ch) ?? 0) + 1); + let bits = 0; + for (const count of counts.values()) { + const p = count / text.length; + bits -= p * Math.log2(p); + } + return bits; +} + +const URL_PREFIX = /^[a-z][a-z0-9+.-]*:\/\//i; + +// A bare value that is really code: a call (getToken(), localStorage.getItem('x');) or a member chain (req.body.token). +const CALL_EXPRESSION = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)*\(/; +const MEMBER_CHAIN = /^[A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)+;?$/; +const IDENTIFIER_PART = /^(?:[a-z_$][a-z0-9_$]*|[A-Z_][A-Z0-9_]*|[a-z]+(?:[A-Z][a-z0-9]*)+)$/; +function looksLikeExpression(value) { + if (CALL_EXPRESSION.test(value)) return true; + return MEMBER_CHAIN.test(value) && value.replace(/;$/, '').split('.').every((part) => IDENTIFIER_PART.test(part)); +} + +// kebab-case or snake_case phrases ("test-signing-key-for-ci!!", "fishCalcState_v2"): every part is a word +// (lower-case, Capitalized, camelCase or ALL-CAPS, with an optional small number) or a bare/version number. +const WORD_PART = /^\d{0,4}(?:[a-z]{2,}(?:[A-Z][a-z]+)*|[A-Z][a-z]+(?:[A-Z][a-z]+)*|[A-Z]{2,})\d{0,4}$/; +const NUMBER_PART = /^v?\d{1,4}$/i; +function isWordIdentifier(value) { + const parts = value.replace(/[!?.]+$/, '').split(/[-_.:/]+/).filter(Boolean); + return parts.length >= 2 && parts.every((p) => NUMBER_PART.test(p) || WORD_PART.test(p)); +} + +// Average length of the word-like pieces of a value. CamelCase identifiers score 4+, random mixed-case runs about 2. +function averagePieceLength(value) { + const pieces = value.match(/[A-Z]?[a-z]+|[A-Z]+(?![a-z])|\d+/g) ?? []; + return pieces.length === 0 ? 0 : value.length / pieces.length; +} + +/** + * A whitespace-free literal that looks machine-generated. Digit-only values need 20+ digits and + * hex-only values 20+ characters (short ones are ids and hashes). Word-like values are excluded: + * kebab-case phrases, CamelCase identifiers and "word + number" passwords such as admin1234567 have long + * letter runs, while random text averages about 2 characters per run. Digit-free single-case values + * (a-z only) must be 24+ characters with high entropy. + */ +function looksRandom(value, { minLength, minEntropy }) { + if (value.length < minLength || /\s/.test(value)) return false; + if (URL_PREFIX.test(value) || isPlaceholder(value)) return false; + if (/^\d+$/.test(value)) return value.length >= 20; + if (/^[0-9a-f]+$/i.test(value)) return value.length >= Math.max(minLength, 20); + if (isWordIdentifier(value)) return false; + const hasDigit = /\d/.test(value); + const mixedCase = /[a-z]/.test(value) && /[A-Z]/.test(value); + const entropy = shannonEntropy(value); + if (!hasDigit && !mixedCase) return value.length >= Math.max(minLength, 24) && entropy >= Math.max(minEntropy, 3.6); + if (averagePieceLength(value) >= (hasDigit ? 4.0 : 3.0)) return false; + return entropy >= minEntropy; +} + +const GATES = { + configWeak: { minLength: 12, minEntropy: 3.0 }, + codeStrong: { minLength: 12, minEntropy: 3.1 }, + codeWeak: { minLength: 20, minEntropy: 3.6 }, +}; +const MIN_STRONG_CONFIG_LENGTH = 8; + +// Words that appear in UI and error sentences ("Invalid or expired token", "Passwords do not match") far more often +// than in a passphrase: function words plus the vocabulary of validation and sign-in messages. Message catalogs +// (en.json, messages.yml) use keys such as password/token/apiKey a lot. Credential-like nouns (password, token, secret) +// are deliberately NOT here: they say nothing about whether the text is a sentence. +const PROSE_WORDS = new Set([ + 'is', 'are', 'was', 'be', 'been', 'not', 'no', 'do', 'does', 'did', 'the', 'a', 'an', 'or', 'and', 'of', 'to', + 'for', 'in', 'on', 'at', 'by', 'as', 'if', 'it', 'its', 'this', 'that', 'these', 'those', 'your', 'you', 'my', + 'please', 'must', 'should', 'cannot', 'can', 'will', 'has', 'have', 'least', 'most', 'than', 'below', 'above', + 'enter', 'choose', 'select', 'type', 'invalid', 'expired', 'missing', 'required', 'incorrect', 'match', + 'matches', 'characters', 'forgot', 'reset', 'confirm', 'wrong', 'empty', 'too', 'short', 'long', 'weak', + 'we', 'our', 'us', 'they', 'their', 'them', 'he', 'she', 'his', 'her', 'so', 'but', 'then', 'when', 'while', + 'because', 'until', 'after', 'before', 'over', 'under', 'out', 'up', 'off', 'yet', 'still', 'also', 'only', 'just', + 'now', 'any', 'all', 'some', 'each', 'every', 'both', 'more', 'less', 'other', 'another', 'such', 'what', 'which', + 'who', 'how', 'why', 'where', 'there', 'am', 'were', 'being', 'had', 'having', 'would', 'could', 'may', 'might', + 'shall', 'let', 'get', 'got', 'make', 'made', 'use', 'used', 'using', 'need', 'needs', 'needed', 'unable', + 'failed', 'error', 'try', 'again', 'later', 'sign', 'with', 'from', 'into', 'about', 'sent', 'check', 'contact', + 'support', 'session', 'log', 'logged', 'expire', 'expires', 'click', 'link', 'email', 'address', 'provided', + 'provide', 'valid', 'value', 'field', 'set', 'updated', 'saved', 'changed', 'successfully', 'successful', + 'new', 'old', 'current', 'first', 'last', 'name', 'account', 'user', 'requested', 'request', 'expected', + 'copy', 'paste', 'one', 'number', 'symbol', 'contain', 'contains', 'found', 'must', 'be', 'at', 'least', +]); + +// Words that make a sentence read as DOCUMENTATION about a credential ("the password you chose during setup", "see the +// deployment guide", "ask the team lead"): instructions and references, not a secret. A passphrase written as a sentence +// of function words ("This is the way.") has none of them. +const DOC_CUE_WORDS = new Set([ + 'see', 'ask', 'refer', 'docs', 'documentation', 'readme', 'guide', 'step', 'steps', 'setup', 'install', 'installed', + 'configure', 'configured', 'configuration', 'generate', 'generated', 'obtain', 'retrieve', 'contact', 'admin', + 'administrator', 'team', 'vault', 'dashboard', 'console', 'portal', 'environment', 'variable', 'variables', 'runtime', + 'deployment', 'deploy', 'chose', 'chosen', 'choose', 'whatever', 'same', 'actual', 'real', 'random', 'string', + 'provided', 'supplied', 'stored', 'store', 'set', 'database', 'value', 'manager', 'instructions', 'above', 'below', +]); +// "the password", "a random token", "your API key": a determiner, up to two words, then a credential noun. +const CREDENTIAL_NOUN_PHRASE = + /(?:^|[^A-Za-z])(?:the|a|an|your|this|that|its|any|each|every|new|correct|current)[ \t]+(?:[A-Za-z-]+[ \t]+){0,2}(?:password|passphrase|passwd|token|secret|key|credentials?)(?![A-Za-z])/i; + +function hasDocumentationCue(text) { + if (CREDENTIAL_NOUN_PHRASE.test(text)) return true; + return text + .toLowerCase() + .split(/[^a-z]+/) + .some((word) => DOC_CUE_WORDS.has(word)); +} + +// A bare number in a sentence is prose only in a counting context ("at least 8 characters", "step 3", "3 attempts"); +// "the horse is 123" is a passphrase with a number in it. +const NUMBER_LEAD_WORDS = new Set([ + 'step', 'steps', 'least', 'most', 'than', 'minimum', 'maximum', 'min', 'max', 'over', 'under', 'exactly', 'between', + 'within', 'section', 'part', 'page', 'version', 'line', 'chapter', 'item', 'option', 'top', 'next', 'last', 'every', + 'each', 'first', +]); +const NUMBER_UNIT_WORDS = new Set([ + 'character', 'characters', 'chars', 'char', 'digit', 'digits', 'letter', 'letters', 'symbol', 'symbols', 'number', + 'numbers', 'word', 'words', 'byte', 'bytes', 'kb', 'mb', 'minute', 'minutes', 'hour', 'hours', 'day', 'days', + 'second', 'seconds', 'ms', 'time', 'times', 'attempt', 'attempts', 'item', 'items', 'entries', 'percent', +]); +const plainWord = (piece) => (piece ?? '').replace(/^[^A-Za-z]+|[^A-Za-z]+$/g, '').toLowerCase(); + +const MESSAGE_CATALOG_DIRS = /(?:^|\/)(?:i18n|l10n|locales?|_locales|lang|langs|languages?|messages?|translations?|intl|strings|res\/values(?:-[A-Za-z0-9]+)*)(?:\/|$)/; +// Segments are [A-Za-z0-9]+ after ONE separator char: the separator class and the segment class must not overlap, +// or a hostile file name ('i18n-' + '--' x 30) backtracks exponentially (CodeQL js/redos; this runs on tracked paths). +const MESSAGE_CATALOG_FILE = + /^(?:(?:messages?|strings|translations?|locales?|i18n|l10n|errors?)(?:[._-][A-Za-z0-9]+)*|[a-z]{2,3}(?:[-_][A-Za-z]{2,4})?)\.(?:json|jsonc|json5|ya?ml|properties|po|pot|xlf|xliff|toml|ini|arb|resx|strings|xml)$/i; + +/** Is this path a message catalog (i18n, locales, messages, en.json, ...), where UI sentences under key names are normal? */ +function isMessageCatalogPath(filePath) { + const normalized = filePath.split(path.sep).join('/').toLowerCase(); + return MESSAGE_CATALOG_DIRS.test(`/${normalized}`) || MESSAGE_CATALOG_FILE.test(path.posix.basename(normalized)); +} + +// Whitespace-separated pieces of a sentence: a word in any script (apostrophes and hyphens inside, punctuation +// around), a short number ("at least 8 characters", "6-digit"), or a translation placeholder (%s, {name}, {{count}}, :attr). +const SENTENCE_WORD = /^["'(\[“‘]*(?:\p{L}[\p{L}'’-]*|\d{1,3}-\p{L}+)["')\]”’]*[.,;:!?…]*$/u; +const SENTENCE_NUMBER = /^["'(\[]*\d{1,4}[)"'.,;:%]*$/; +const SENTENCE_PLACEHOLDER = /^["'(]*(?:%(?:\d\$)?[sd]|\{\{?[\w.-]{1,30}\}\}?|\{\d{1,2}\}|:[a-z_]{1,30})[)"'.,;:!?]*$/; + +/** + * Is this quoted, whitespace-containing text a natural-language sentence (a UI or error message) rather than a passphrase? + * Deliberately strict, because a passphrase such as "correct horse battery and staple" is made of ordinary words: + * - every piece must be a plain word, a short number or a translation placeholder. A piece that mixes letters + * with digits or symbols ("Passw0rd!", "123!") is credential-shaped, so the whole text is not prose; + * - most of the words must be sentence vocabulary (PROSE_WORDS): at least half when the text is written like a + * sentence (capital first letter or closing punctuation), at least two thirds when it is not; + * - text with non-ASCII letters made of plain words counts as prose in another language (message catalogs); + * - a bare number that counts nothing ("the horse is 123") is a non-prose word. + * Being sentence-like is necessary, not sufficient: isPhraseSecret exempts it only in a message catalog path (i18n, + * locales, messages, en.json, ...) or when it reads as documentation (hasDocumentationCue). + * Tradeoff: a passphrase that is a sentence and also mentions "the password" or an instruction word passes; a UI sentence + * outside a message catalog that has neither is reported (add the allow marker, or a placeholder value), and so is an + * ASCII sentence in a language other than English. + */ +function looksLikeSentence(text) { + const trimmed = text.trim(); + const pieces = trimmed.split(/\s+/); + if (pieces.length < 2) return false; + let counted = 0; + let prose = 0; + for (let i = 0; i < pieces.length; i += 1) { + const piece = pieces[i]; + if (SENTENCE_PLACEHOLDER.test(piece)) continue; + if (SENTENCE_NUMBER.test(piece)) { + // A number counts against the sentence unless it counts something ("at least 8 characters", "step 3"). + const counting = NUMBER_LEAD_WORDS.has(plainWord(pieces[i - 1])) || NUMBER_UNIT_WORDS.has(plainWord(pieces[i + 1])) || /%/.test(piece); + if (!counting) counted += 1; + continue; + } + if (!SENTENCE_WORD.test(piece)) return false; + counted += 1; + if (PROSE_WORDS.has(plainWord(piece))) prose += 1; + } + if (counted === 0) return false; + if (pieces.length >= 3 && /[^\x00-\x7f]/.test(trimmed)) return true; + const sentenceShaped = /^["'(“‘]*[A-Z]/.test(trimmed) || /[.!?…:]["')”’]*$/.test(trimmed); + return sentenceShaped ? prose * 2 >= counted : prose * 3 >= counted * 2; +} + +/** + * Documentation about a credential, outside a message catalog ("the password you chose during setup", "see the + * deployment guide", "set via environment variable at runtime"): three or more plain words (no mixed letter/digit/symbol + * piece, no stray number), at least one instruction or reference cue, and a third or more of them sentence vocabulary. + */ +function looksLikeDocumentation(text) { + const pieces = text.trim().split(/\s+/); + if (pieces.length < 3 || !hasDocumentationCue(text)) return false; + let counted = 0; + let prose = 0; + for (let i = 0; i < pieces.length; i += 1) { + const piece = pieces[i]; + if (SENTENCE_PLACEHOLDER.test(piece)) continue; + if (SENTENCE_NUMBER.test(piece)) { + if (!(NUMBER_LEAD_WORDS.has(plainWord(pieces[i - 1])) || NUMBER_UNIT_WORDS.has(plainWord(pieces[i + 1])) || /%/.test(piece))) return false; + continue; + } + if (!SENTENCE_WORD.test(piece)) return false; + counted += 1; + if (PROSE_WORDS.has(plainWord(piece))) prose += 1; + } + return counted >= 3 && prose * 3 >= counted; +} + +/** + * A value with whitespace in it (a quoted passphrase, or the rest of a YAML/ini line). It is a finding when a + * word in it is random-looking, or, for a strong name, when it is not a sentence and is long enough. + */ +function isPhraseSecret({ kind, text, minLength = MIN_STRONG_CONFIG_LENGTH, catalog = false }) { + const words = text + .split(/\s+/) + .map((w) => w.replace(/^[^A-Za-z0-9]+|[.,;:!?)]+$/g, '')) + .filter(Boolean); + if (words.some((w) => looksRandom(w, GATES.configWeak))) return true; + // In a message catalog (its whole job is UI text) a sentence is exempt. Anywhere else only text that reads as + // documentation about the credential is: a sentence of common words ("This is the way.") is a passphrase until an + // allow marker says otherwise. + if (catalog ? looksLikeSentence(text) : looksLikeDocumentation(text)) return false; + return kind === 'strong' && text.length >= minLength; +} + +const stripQuotes = (text) => text.trim().replace(/^["'`]+|["'`]+$/g, ''); + +// Query and fragment parameter names that carry a credential: sig, signature, X-Amz-Signature, token, access_token, +// key, api_key, secret, password, auth, ... +const CREDENTIAL_PARAM = /(?:^|[-_.])(?:sig|signature|token|secret|key|apikey|password|passwd|pwd|auth|authorization|jwt|bearer|sas)$/i; + +const safeDecode = (text) => { + try { + return decodeURIComponent(text); + } catch { + return text; + } +}; + +// A path segment written as a template ({token}, :token, , ${TOKEN}) stands for a value, it is not one. +const isTemplateSegment = (segment) => /[{}<>$]|^:/.test(segment) || isPlaceholder(segment); + +/** + * Does a URL that is the VALUE of a strong secret name (API_TOKEN, WEBHOOK_SECRET, ...) carry a credential itself? + * Signed URLs (?sig=..., X-Amz-Signature), webhook URLs (/services/T000/B000/) and token-as-username URLs + * (https://@host) are bearer credentials. A URL whose parts are all ordinary words, ids and template + * placeholders is just an address. Weak names (TOKEN_URL, TOKEN_ENDPOINT) never reach this: they name an endpoint. + * The password of user:password@ is url-password's business, not judged here. + */ +function urlCarriesCredential(url) { + const rest = url.replace(URL_PREFIX, ''); + const authorityEnd = rest.search(/[/?#]/); + const authority = authorityEnd === -1 ? rest : rest.slice(0, authorityEnd); + const afterAuthority = authorityEnd === -1 ? '' : rest.slice(authorityEnd); + const at = authority.lastIndexOf('@'); + if (at > 0) { + const userinfo = authority.slice(0, at); + if (!userinfo.includes(':') && userinfo.length >= 8 && !isTemplateSegment(userinfo) && !isWordIdentifier(userinfo)) return true; + } + const hashAt = afterAuthority.indexOf('#'); + const beforeFragment = hashAt === -1 ? afterAuthority : afterAuthority.slice(0, hashAt); + const fragment = hashAt === -1 ? '' : afterAuthority.slice(hashAt + 1); + const queryAt = beforeFragment.indexOf('?'); + const pathPart = queryAt === -1 ? beforeFragment : beforeFragment.slice(0, queryAt); + const query = queryAt === -1 ? '' : beforeFragment.slice(queryAt + 1); + for (const segment of pathPart.split('/')) { + const decoded = safeDecode(segment); + if (decoded.length >= 16 && !isTemplateSegment(decoded) && looksRandom(decoded, { minLength: 16, minEntropy: 3.0 })) return true; + } + for (const pair of `${query}&${fragment}`.split(/[&;]/)) { + if (pair === '') continue; + const eq = pair.indexOf('='); + const name = eq === -1 ? '' : safeDecode(pair.slice(0, eq)); + const value = safeDecode(eq === -1 ? pair : pair.slice(eq + 1)); + if (value === '' || isTemplateSegment(value)) continue; + if (CREDENTIAL_PARAM.test(name)) { + if (value.length >= 8 && !isWordIdentifier(value)) return true; + } else if (looksRandom(value, { minLength: 20, minEntropy: 3.5 })) { + return true; + } + } + return false; +} + +/** + * Decide whether a value assigned to a secret-like name is a finding. + * @param {{kind: 'strong'|'weak', value: string, quoted: boolean, separator: string, mode: string, minLength?: number}} input + * `minLength` is the shortest value a strong name may hold (8; credential files such as .npmrc use 4). + */ +function isSecretValue({ kind, value, quoted, separator, mode, minLength = MIN_STRONG_CONFIG_LENGTH, catalog = false }) { + let text = value; + if (!quoted) { + if (mode === 'code') return false; // a bare token in source code is an expression, not a literal + if (looksLikeExpression(text)) return false; + if (mode === 'prose') text = text.replace(/[`*)>\].,;:!?]+$/, ''); + } else { + text = text.trim(); // a quoted value is the whole quoted string, spaces included + } + // Shell/Compose/env files: ${VAR:-literal} hides a real default inside the expansion, and a literal can sit + // right next to a reference (${A}literal). Operands are judged as whole strings. + if (mode === 'config' && text.includes('${')) { + const operand = (literal) => + isSecretValue({ kind, value: stripQuotes(literal), quoted: true, separator: '=', mode, minLength, catalog }); + if (expansionLiterals(text).some(operand)) return true; + const glued = stripQuotes(withoutExpansions(text)); + if (glued === '') return false; + return URL_PREFIX.test(glued) ? kind === 'strong' && urlCarriesCredential(glued) : looksRandom(glued, GATES.configWeak); + } + if (URL_PREFIX.test(text)) { + // A URL value is an address for weak names (TOKEN_URL, TOKEN_ENDPOINT) and for URLs with nothing credential-like in them, + // but a bearer credential when it sits in a strong name and a part of it is one (signed URL, webhook path token). + return kind === 'strong' && !/\s/.test(text) && urlCarriesCredential(text); + } + if (text === '' || isPlaceholder(text)) return false; + if (/\s/.test(text)) { + // Only a quoted value (or a whole-line value) may contain spaces; in code a spaced string is text. + return quoted && mode !== 'code' && isPhraseSecret({ kind, text, minLength, catalog }); + } + if (mode === 'code') return looksRandom(text, kind === 'strong' ? GATES.codeStrong : GATES.codeWeak); + // Prose such as "Token: something" is common, so an unquoted `name: word` needs a random-looking word. + const proseColon = mode === 'prose' && !quoted && separator !== '='; + if (kind === 'strong' && !proseColon) return text.length >= minLength; + return looksRandom(text, GATES.configWeak); +} + +// YAML node properties (tags and anchors) in front of a scalar: `!!str`, `!vault`, `&default`, one or more, then a blank. +const YAML_NODE_PROPERTIES = /(?:(?:![^\s]{0,60}|&[A-Za-z0-9_-]{1,60})[ \t]+)+(?=\S)/y; + +/** Index of the "}" that closes the "${" at `open`, or -1. Nesting-aware, linear. */ +function closingBrace(text, open) { + let depth = 0; + for (let i = open; i < text.length; i += 1) { + if (text[i] === '$' && text[i + 1] === '{') { + depth += 1; + i += 1; + } else if (text[i] === '}') { + depth -= 1; + if (depth === 0) return i; + } + } + return -1; +} + +/** `text` with every ${...} removed. An unbalanced one is dropped up to the end of the text (its operand is judged separately). */ +function withoutExpansions(text) { + let out = ''; + let i = 0; + for (let open = text.indexOf('${'); open !== -1; open = text.indexOf('${', i)) { + const end = closingBrace(text, open); + out += text.slice(i, open); + if (end === -1) return out; + i = end + 1; + } + return out + text.slice(i); +} + +const MAX_EXPANSION_DEPTH = 8; + +/** + * Literal default/alternate operands of shell parameter expansions (:- - := = :+ +), nested ones included. + * A pure ${VAR}, $VAR or ${VAR:?message} has none, so it is never a candidate secret. + * Fails closed: past the depth cap, or in an unterminated expansion, the remaining literal text is a candidate. + */ +function expansionLiterals(text, depth = 0) { + const literals = []; + if (depth > MAX_EXPANSION_DEPTH) { + const flat = text.replace(/\$\{[A-Za-z_][A-Za-z0-9_]{0,1023}(?::?[-=+?])?|\}/g, '').trim(); + if (flat !== '') literals.push(flat); + return literals; + } + let i = 0; + for (let open = text.indexOf('${', i); open !== -1; open = text.indexOf('${', i)) { + const end = closingBrace(text, open); + const inner = text.slice(open + 2, end === -1 ? text.length : end); + const operation = /^[A-Za-z_][A-Za-z0-9_]{0,1023}:?[-=+]([\s\S]*)$/.exec(inner); + if (operation) { + const literal = withoutExpansions(operation[1]).trim(); + if (literal !== '') literals.push(literal); + literals.push(...expansionLiterals(operation[1], depth + 1)); + } + if (end === -1) break; + i = end + 1; + } + return literals; +} + +// End (exclusive) of a shell word starting at `start`: stops at whitespace outside any ${...}, or at the end of the line. +function bareShellWordEnd(input, start) { + let depth = 0; + let i = start; + for (; i < input.length && i - start < 8192; i += 1) { + const ch = input[i]; + if (ch === '\n') break; + if (ch === '$' && input[i + 1] === '{') { + depth += 1; + i += 1; + } else if (ch === '}' && depth > 0) { + depth -= 1; + } else if (depth === 0 && (ch === ' ' || ch === '\t' || ch === '\r')) { + break; + } + } + return i; +} + +// Files whose unquoted values run to the end of the line (YAML plain scalars, ini, properties, dotenv). +const REST_OF_LINE_EXTENSIONS = new Set(['.yml', '.yaml', '.ini', '.cfg', '.conf', '.config', '.properties', '.toml', '.env']); +const REST_OF_LINE_FORMATS = ['npmrc', 'pypirc', 'awscreds', 'mycnf', 's3cfg', 'wgetrc', 'terraformrc']; +function valueRunsToEndOfLine(filePath) { + const base = path.posix.basename(filePath).toLowerCase(); + if (REST_OF_LINE_EXTENSIONS.has(path.posix.extname(base)) || isPropertiesPath(filePath) || base === '.env' || base.startsWith('.env.') || base.endsWith('.env')) return true; + const formats = credentialFormats(filePath); + return REST_OF_LINE_FORMATS.some((tag) => formats.has(tag)); +} + +/** + * Extra candidate values for an unquoted `name: value` / `name = value`, where the first whitespace-free token + * (the regex capture) is not the whole value. Returns [{ value, end }] with `end` the index the value reaches. + * - the rest of the line, comment removed (a plain scalar or ini value with spaces) + * - the indented lines of a YAML block scalar (`|`, `>`, `|-`, `>-`, ...) + */ +function unquotedContinuations(ctx, input, matchIndex, tokenEnd, token) { + // Line boundaries come from the per-file newline index (binary search), never from a scan: a one-line file with + // thousands of matches must stay linear. + const lineEnd = ctx.lineEnd(tokenEnd); + const results = []; + if (/^[|>][-+0-9]*$/.test(token)) { + const lineStart = ctx.lineStart(matchIndex); + const keyIndent = /^[ \t]*/.exec(input.slice(lineStart, matchIndex + 1))[0].length; + let cursor = lineEnd + 1; + const collected = []; + let end = lineEnd; + for (let n = 0; n < 60 && cursor <= input.length; n += 1) { + let next = input.indexOf('\n', cursor); + if (next === -1) next = input.length; + const line = input.slice(cursor, next).replace(/\r$/, ''); + if (line.trim() !== '') { + if (/^[ \t]*/.exec(line)[0].length <= keyIndent) break; + collected.push(line.trim()); + end = next; + } + if (next >= input.length) break; + cursor = next + 1; + } + for (const line of collected) results.push({ value: line, end }); + if (collected.length > 1) results.push({ value: collected.join(' '), end }); + return results; + } + if (!ctx.runsToEndOfLine || /^[!&*'"`{[]/.test(token) || token.endsWith(',')) return results; + const rest = input.slice(tokenEnd, Math.min(lineEnd, tokenEnd + 400)); + if (!/^[ \t]+[^\s#]/.test(rest)) return results; + const whole = (token + rest).replace(/\r$/, '').replace(/[ \t]+#.*$/, '').trim(); + if (/[{}[\]]/.test(whole)) return results; + results.push({ value: whole, end: tokenEnd + rest.length }); + return results; +} + +// --------------------------------------------------------------------------- +// File modes +// --------------------------------------------------------------------------- + +const PROSE_EXTENSIONS = new Set(['.md', '.mdx', '.txt', '.rst', '.adoc']); +const CONFIG_EXTENSIONS = new Set([ + '.env', '.ini', '.cfg', '.conf', '.config', '.properties', '.toml', '.yml', '.yaml', '.json', '.jsonc', + '.json5', '.sh', '.bash', '.zsh', '.fish', '.ksh', '.csh', '.tcsh', '.bat', '.cmd', '.tf', '.tfvars', '.hcl', '.example', + '.sample', '.template', '.dist', '.cnf', '.tfstate', '.kubeconfig', '.acl', +]); +// XML configuration formats: Maven settings.xml and pom.xml, Ant, Tomcat server.xml, Android strings.xml, .NET +// web.config / app.config (.config is above), MSBuild (.csproj, .props, .targets), .plist, .resx, .wsdl. These hold settings +// the same way an ini or YAML file does, so they get configuration-value semantics (a quoted value with spaces or a +// passphrase is a secret, not text). .svg, .html and .xhtml stay in code mode: they are markup, not settings. +const XML_CONFIG_EXTENSIONS = new Set([ + '.xml', '.csproj', '.vbproj', '.fsproj', '.props', '.targets', '.plist', '.resx', '.wsdl', '.nuspec', '.pubxml', '.settings', + // MSBuild siblings (any *proj is added by isXmlConfigPath), Azure service configuration, JMeter, Apple profiles and + // entitlements, IDE and analyser settings, Windows packaging, Maven POM files. + '.projitems', '.cscfg', '.csdef', '.jmx', '.mobileconfig', '.entitlements', '.iml', '.launch', '.ruleset', '.appxmanifest', + '.wxs', '.wxi', '.pom', '.jnlp', +]); +// Markup and schema formats stay in code mode on purpose: .svg .html .htm .xhtml .xsl .xslt .xaml .xsd .rss .atom .kml hold +// content, styling or structure, not settings, and a label or a form field there is not a credential. +// A template or backup suffix (settings.xml.template, web.xml.erb, pom.xml.bak) does not change what the file is. +const TEMPLATE_SUFFIX = /\.(?:template|dist|sample|example|erb|j2|jinja2?|tpl|tmpl|bak|orig|old|default|in)$/i; + +/** The file name without trailing template or backup suffixes (at most two), lower-cased. */ +function baseWithoutTemplateSuffix(base) { + let name = base.toLowerCase(); + for (let i = 0; i < 2 && TEMPLATE_SUFFIX.test(name) && name.replace(TEMPLATE_SUFFIX, '') !== ''; i += 1) name = name.replace(TEMPLATE_SUFFIX, ''); + return name; +} + +/** True for XML configuration files (see XML_CONFIG_EXTENSIONS) and .NET *.config files. */ +export function isXmlConfigPath(filePath) { + const base = path.posix.basename(filePath.split(path.sep).join('/')); + const ext = path.posix.extname(baseWithoutTemplateSuffix(base)); + return XML_CONFIG_EXTENSIONS.has(ext) || ext === '.config' || /^\.[a-z]*proj$/.test(ext); +} + +const CONFIG_BASENAMES = new Set([ + '.npmrc', '.yarnrc', '.netrc', '.pgpass', '.envrc', 'makefile', 'gnumakefile', 'procfile', 'credentials', 'config', + // Shell start-up files have no extension: `export API_TOKEN=...` in them is a setting, not a code expression. + '.bashrc', '.bash_profile', '.bash_login', '.bash_aliases', '.profile', '.zshrc', '.zshenv', '.zprofile', '.zlogin', + '.kshrc', '.cshrc', '.tcshrc', '.login', 'fish_variables', + // Whitespace-delimited service settings without an extension (see the config-directive-secret rule). + '.htaccess', 'msmtprc', '.msmtprc', 'mpoprc', '.mpoprc', 'fetchmailrc', '.fetchmailrc', +]); + +const WHOLE_FILE_SECRET_NAME = /^(?:\.?erlang\.cookie|(?:mongo(?:db)?[._-])?keyfile|mongo(?:db)?\.key)$/; +const NETRC_NAME = /(?:^|[._-])netrc(?:$|[._-])/; +const PGPASS_NAME = /(?:^|[._-])pgpass(?:$|[._-])/; +const GITCRED_NAME = /(?:^|[._-])git-credentials(?:$|[._-])/; +const DOCKER_NAME = /(?:^|[._-])(?:dockercfg|dockerconfigjson|dockerconfig|docker-config)(?:$|[._-])/; + +/** + * Which native credential-file formats a path is, by its base name and directories (all lower-case tags): + * netrc pgpass gitcred npmrc pypirc awscreds docker kube htpasswd mycnf s3cfg terraformrc curlrc wgetrc vault + * A path can carry several. These files have their own syntax (`password `, host:port:db:user:password, + * URL lines, `_authToken=`, user:hash) and are scanned by the credential-file rule, with lower value-length limits. + */ +export function credentialFormats(filePath) { + const normalized = filePath.split(path.sep).join('/').toLowerCase(); + const base = path.posix.basename(normalized); + const dirs = `/${normalized}`; + const tags = new Set(); + // Backups and variants reach a repository under names like netrc.txt, .netrc.bak, .netrc.prod, dot-netrc, pgpass.local: + // the name is matched as a whole word (separated by . _ or -), not as an exact base name. + if (NETRC_NAME.test(base)) tags.add('netrc'); + if (PGPASS_NAME.test(base)) tags.add('pgpass'); + if (GITCRED_NAME.test(base)) tags.add('gitcred'); + if (/^\.?(?:npmrc|yarnrc)(?:\.|$)/.test(base)) tags.add('npmrc'); + if (/^\.?pypirc(?:\.|$)/.test(base)) tags.add('pypirc'); + if (base === 'credentials' || (base === 'config' && dirs.includes('/.aws/')) || base === '.aws-credentials') tags.add('awscreds'); + if (DOCKER_NAME.test(base) || dirs.includes('/.docker/')) tags.add('docker'); + if (base === 'kubeconfig' || base.endsWith('.kubeconfig') || base.startsWith('kubeconfig.') || dirs.includes('/.kube/')) tags.add('kube'); + if (/^\.?ht(?:passwd|digest)(?:\.|$)/.test(base)) tags.add('htpasswd'); + if (/^\.?(?:my|mylogin|mysql)\.cnf$/.test(base)) tags.add('mycnf'); + if (base === '.s3cfg' || base === 's3cfg' || base === '.boto' || base === 'boto.cfg') tags.add('s3cfg'); + if (base === '.terraformrc' || base === 'terraform.rc' || base.endsWith('.tfrc.json') || base.endsWith('.tfrc') || base.includes('.tfstate')) tags.add('terraformrc'); + if (/^[._]?curlrc$/.test(base)) tags.add('curlrc'); + if (base === '.wgetrc' || base === 'wgetrc') tags.add('wgetrc'); + // Files whose whole content is the secret: a Vault token, a MongoDB replica-set key file, an Erlang (RabbitMQ) cookie. + if (base === '.vault-token' || base === 'vault-token' || WHOLE_FILE_SECRET_NAME.test(base)) tags.add('vault'); + return tags; +} + +// Formats in which every `name = value` is a credential setting: the value-length limit drops from 8 to 4 and +// auth-style names (auth, npmAuthIdent, client-key-data) count as secret names. +const STRICT_CREDENTIAL_FORMATS = ['npmrc', 'pypirc', 'awscreds', 'docker', 'kube', 'mycnf', 's3cfg', 'terraformrc', 'wgetrc', 'curlrc']; +const CREDENTIAL_FILE_MIN_LENGTH = 4; +const CREDENTIAL_FILE_NAMES = new Set(['auth', 'authident', 'npmauthident', 'clientkeydata', 'clientkey', 'authorization', 'basicauth']); + +// Dovecot keeps settings in dovecot.conf, dovecot-sql.conf.ext, dovecot-ldap.conf.ext: `.ext` alone says nothing about a file. +const DOVECOT_CONFIG = /^dovecot[a-z0-9._-]{0,60}\.conf(?:\.ext)?$/; + +/** + * 'prose' Markdown and text: docs paste real values into code fences. + * 'config' env files, YAML, JSON, INI, shell, Terraform, Makefile, credential files, ...: bare KEY=value lines. + * 'code' everything else (JS, TS, Python, SQL, HTML, ...): only quoted, random-looking literals. + */ +/** + * Is this a Java properties file: `app.properties`, or one with an environment or backup suffix (`app.properties.local`, + * `application.properties.prod`, `db.properties.bak`)? Its lines are `key value`, `key=value` or `key:value`. + */ +export function isPropertiesPath(filePath) { + const base = path.posix.basename(filePath.split(path.sep).join('/')).toLowerCase(); + const at = base.lastIndexOf('.properties'); + if (at === -1) return false; + const rest = base.slice(at + '.properties'.length); + return rest === '' || (rest.length <= 33 && rest[0] === '.' && /^[a-z0-9_-]+$/.test(rest.slice(1))); +} + +export function fileMode(filePath) { + const base = path.posix.basename(filePath.split(path.sep).join('/')).toLowerCase(); + const ext = path.posix.extname(base); + if (PROSE_EXTENSIONS.has(ext)) return 'prose'; + if ( + base === '.env' || + base.startsWith('.env.') || + base.includes('.env.') || + base.endsWith('.env') || + base.startsWith('env.') || + base.startsWith('docker-compose') || + base.startsWith('dockerfile') || + DOVECOT_CONFIG.test(base) || + CONFIG_BASENAMES.has(base) || + CONFIG_EXTENSIONS.has(ext) || + XML_CONFIG_EXTENSIONS.has(ext) || + isXmlConfigPath(filePath) || + CONFIG_EXTENSIONS.has(path.posix.extname(baseWithoutTemplateSuffix(base))) || + isPropertiesPath(filePath) || + credentialFormats(filePath).size > 0 + ) { + return 'config'; + } + return 'code'; +} + +// --------------------------------------------------------------------------- +// Rules +// --------------------------------------------------------------------------- + +// One or more line breaks (real or JSON-escaped). Unambiguous on purpose, to avoid regex backtracking blow-ups. +const PEM_SEPARATOR = String.raw`(?:[ \t]*(?:\\r|\r)?(?:\\n|\n))+[ \t]*`; +const SECRET_HINT = /secret|passw|pwd|pass|token|credential|salt|pepper|key|auth/i; +const ENV_ROOT = String.raw`(?:process\.env|import\.meta\.env)`; +// A "/" in a URL, or the same "/" escaped for JSON (\/). +const SLASH = String.raw`\\?\/`; + +// `sign(payload, KEY, ...)`: is the text before the literal exactly one top-level argument? +function isSecondArgument(prefix) { + let depth = 0; + for (const ch of prefix) { + if (ch === '(' || ch === '[' || ch === '{') depth += 1; + else if (ch === ')' || ch === ']' || ch === '}') depth -= 1; + else if (ch === ',' && depth === 0) return false; + if (depth < 0) return false; + } + return depth === 0; +} + +// The text of the match's own line, bounded to 200 characters before and 400 after so a huge single line stays cheap. +function nearbyLineText(m) { + const before = m.input.slice(Math.max(0, m.index - 200), m.index); + const after = m.input.slice(m.index + m[0].length, m.index + m[0].length + 400); + const lineBreak = after.indexOf('\n'); + return before.slice(before.lastIndexOf('\n') + 1) + m[0] + (lineBreak === -1 ? after : after.slice(0, lineBreak)); +} + +// The value after `name =`, read at a given position: "quoted", 'quoted' or `quoted` (with backslash escapes) +// in groups 1-3, otherwise the bare whitespace-free token in group 4. +const valueAt = (bareMax) => + new RegExp( + String.raw`"((?:[^"\\\n]|\\[\s\S]){0,4096})"|'((?:[^'\\\n]|\\[\s\S]){0,4096})'|\x60((?:[^\x60\\\n]|\\[\s\S]){0,4096})\x60|(\S{1,${bareMax}})`, + 'y', + ); +const NESTED_VALUE_CHARS = 64; +const VALUE_AT = valueAt(4096); +const NESTED_VALUE_AT = valueAt(NESTED_VALUE_CHARS); + +/** Undo backslash escapes inside a quoted value (\" \\ \'), so an escaped quote cannot hide the rest of the string. */ +const unescapeQuoted = (text) => text.replace(/\\\r?\n[ \t]*/g, '').replace(/\\(.)/g, '$1'); + +const QUOTED_ARGUMENT_MAX = 4096; +/** + * The quoted string that starts at `start` (text[start] is a quote) and is the last argument of a call: its body without the quotes + * (`raw`, escapes still in place) and the index after the closing quote (`end`), or null when it is not one complete string on one line + * followed by `)` or `,`. A triple quote (three double or three single quotes) ends at the first run of three; the others end at the first quote that is not escaped. + * One pass over at most QUOTED_ARGUMENT_MAX body characters. + */ +function quotedArgument(text, start) { + const quote = text[start]; + const triple = (quote === '"' || quote === "'") && text[start + 1] === quote && text[start + 2] === quote; + const bodyStart = start + (triple ? 3 : 1); + const limit = Math.min(text.length, bodyStart + QUOTED_ARGUMENT_MAX + 3); + let i = bodyStart; + for (; i < limit; i += 1) { + const c = text[i]; + if (c === '\n') return null; + if (c === '\\') { + if (text[i + 1] === undefined || text[i + 1] === '\n') return null; + i += 1; + } else if (c === quote && (!triple || (text[i + 1] === quote && text[i + 2] === quote))) { + break; + } + } + if (i >= limit || i - bodyStart > QUOTED_ARGUMENT_MAX) return null; + const end = i + (triple ? 3 : 1); + let after = end; + while (after < text.length && after - end < 8 && (text[after] === ' ' || text[after] === '\t')) after += 1; + if (text[after] !== ')' && text[after] !== ',') return null; + return { raw: text.slice(bodyStart, i), end }; +} + +/** + * A password taken from a URL or `curl -u`. A password with no ${...} is judged as a whole. One with an expansion + * is judged like a secret-like assignment: a reference alone (${DB_PASSWORD}) passes, but a literal default + * (${DB_PASSWORD:-hunter2}) or literal text next to a reference (${A}suffix) is a candidate password. + */ +function urlPasswordIsSecret(password) { + if (!password.includes('${')) { + if (isPlaceholder(password)) return false; + // A quoted password may hold spaces (a curl user argument written as one quoted string). The whole argument is judged like a + // quoted passphrase: a placeholder-like first word does not hide the rest, and documentation about a password passes. + return /\s/.test(password) ? isPhraseSecret({ kind: 'strong', text: password }) : true; + } + if (expansionLiterals(password).some((literal) => !isPlaceholder(stripQuotes(literal)))) return true; + const glued = stripQuotes(withoutExpansions(password)); + return glued !== '' && !isPlaceholder(glued); +} + +/** + * The first shell word at `start` (quotes honoured, at most one line and 4096 characters read). + * @returns {{text: string, length: number, quoted: boolean} | null} `length` is how much input the word's source covers + */ +function shellArgument(input, start) { + let end = input.indexOf('\n', start); + if (end === -1 || end - start > 4096) end = Math.min(input.length, start + 4096); + let source = input.slice(start, end); + // The word's source ends at the first unquoted blank. + let quote = null; + let openedAt = -1; + let i = 0; + for (; i < source.length; i += 1) { + const ch = source[i]; + if (quote !== null) { + if (ch === '\\' && (quote === '"' || quote === "'") && i + 1 < source.length) i += 1; + else if (ch === quote) quote = null; + } else if (ch === '"' || ch === "'") { + quote = ch; + openedAt = i; + } else if (ch === '\\') i += 1; + else if (ch === ' ' || ch === '\t' || ch === '\r') break; + } + // A quote that never closes on the line is not a quoted argument: the text stands in a string of the surrounding code + // ('curl -u user:pass', "..."), so the word ends at that quote character. + if (quote !== null) { + source = source.slice(0, openedAt); + i = openedAt; + } + const words = shellWords(source); + if (words.length === 0) return null; + return { text: words[0], length: Math.min(i, source.length), quoted: /["']/.test(source.slice(0, i)) }; +} + +// Commands that take a password on their command line, and the rest of that line (see cliPasswordArguments). +const CLI_PASSWORD_COMMAND = + /(? (i + 1 < words.length && !words[i + 1].startsWith('-') ? words[i + 1] : null); + // A password given as a shell variable ($PW, %PW%) is a reference, not a literal. + const literal = (value) => { + if (value !== null && value !== undefined && !SHELL_REFERENCE.test(value)) found.push(value); + }; + const auth = (value) => { + // user:password; a value without a colon is a bearer token only when the command says so. + const colon = value.indexOf(':'); + if (colon !== -1) found.push(value.slice(colon + 1)); + else if (words.some((w, k) => /^(?:-A|--auth-type)(?:=|$)/.test(w) && /bearer/i.test(w.includes('=') ? w : (words[k + 1] ?? '')))) found.push(value); + }; + for (let i = 1; i < words.length; i += 1) { + const word = words[i]; + const eq = word.indexOf('='); + const flag = eq === -1 ? word : word.slice(0, eq); + const inline = eq === -1 ? null : word.slice(eq + 1); + const value = () => inline ?? next(i); + if (tool === 'curl') { + if (flag === '--pass' || flag === '--proxy-pass') found.push(value() ?? ''); + } else if (tool.startsWith('wget')) { + if (/^--(?:http-|ftp-|proxy-)?password$/.test(flag)) found.push(value() ?? ''); + } else if (/^(?:mysql|mariadb)/.test(tool)) { + if (flag === '--password' && inline !== null) found.push(inline); + else if (/^-p./.test(word) && !word.startsWith('--')) found.push(word.slice(2)); // `-p` alone prompts + } else if (tool.startsWith('mongo')) { + if (flag === '--password') found.push(value() ?? ''); + else if (word === '-p') found.push(next(i) ?? ''); + else if (/^-p./.test(word) && !word.startsWith('--')) found.push(word.slice(2)); + } else if (tool === 'redis-cli' || tool === 'redis6-cli' || tool === 'valkey-cli') { + if (word === '-a' || flag === '--pass') found.push(value() ?? ''); + } else if (tool === 'sshpass') { + if (word === '-p') found.push(next(i) ?? ''); + else if (/^-p./.test(word)) found.push(word.slice(2)); + } else if (tool === 'xh' || tool === 'xhs' || tool === 'http' || tool === 'https') { + if (word === '-a' || flag === '--auth') auth(value() ?? ''); + else if (/^-a./.test(word) && !word.startsWith('--')) auth(word.slice(2)); + } else if (tool === 'smbclient') { + if (word === '-U' || flag === '--user') { + const login = value() ?? ''; // user%password + if (login.includes('%')) found.push(login.slice(login.indexOf('%') + 1)); + } + } else if (tool.startsWith('ldap')) { + if (word === '-w') found.push(next(i) ?? ''); + else if (/^-w./.test(word)) found.push(word.slice(2)); + } else if (tool === 'mosquitto_pub' || tool === 'mosquitto_sub' || tool === 'mosquitto_rr') { + if (word === '-P' || flag === '--pw') literal(value()); + } else if (tool === 'mosquitto_passwd') { + // mosquitto_passwd -b [-c] passwordfile username password (without -b the password is prompted for) + if (/^-[A-Za-z]*b[A-Za-z]*$/.test(word)) { + const operands = words.slice(i + 1).filter((w) => !w.startsWith('-')); + if (operands.length >= 3) literal(operands[2]); + } + } else if (tool === 'htpasswd') { + // htpasswd -b[cmBdps] passwordfile username password; -nb username password + if (/^-[A-Za-z]*b[A-Za-z]*$/.test(word)) { + const operands = words.slice(i + 1).filter((w) => !w.startsWith('-')); + const password = /^-[A-Za-z]*n/.test(word) ? operands[1] : operands[2]; + if (password !== undefined) literal(password); + } + } else if (tool === 'rabbitmqctl') { + // rabbitmqctl add_user USER PASSWORD, change_password USER PASSWORD, authenticate_user USER PASSWORD + if (/^(?:add_user|change_password|authenticate_user)$/.test(word) && i + 2 < words.length) literal(words[i + 2]); + } else if (tool === 'rabbitmqadmin') { + if (word === '-p' || flag === '--password') literal(value()); + } else if (tool === 'sqlcmd') { + if (word === '-P') literal(next(i)); + } else if (tool === 'keytool') { + if (/^-(?:store|key|deststore|destkey|srcstore|srckey)pass$/.test(word)) literal(next(i)); + } else if (tool === 'gpg' || tool === 'gpg2') { + if (flag === '--passphrase') literal(value()); + } else if (tool === 'docker' || tool === 'podman') { + if (words[1] === 'login' && (word === '-p' || flag === '--password')) literal(value()); + } else if (tool === 'az') { + if (words[1] === 'login' && (word === '-p' || flag === '--password')) literal(value()); + } else if (tool === 'vault') { + // vault login TOKEN: the first operand that is not a flag or a key=value option + if (words[1] === 'login' && i === 2 && !word.startsWith('-') && !word.includes('=') && looksRandom(word, GATES.configWeak)) found.push(word); + } + } + return found.filter((password) => password !== ''); +} + +// --------------------------------------------------------------------------- +// Name/value pairs split across fields (JSON, YAML, HCL objects), in any order +// --------------------------------------------------------------------------- + +const PAIR_BACK_CHARS = 1500; // how far before the name field the enclosing "{" or "(" is looked for +const PAIR_FORWARD_CHARS = 1500; // ... and how far after it the enclosing "}" or ")" is looked for +const PAIR_YAML_LINES = 12; // lines above and below the name line that can belong to the same YAML mapping +const PAIR_LINE_CHARS = 2000; // only this much of a YAML line is read +const PAIR_XML_CHARS = 600; // an XML entry (, ) is read up to this size +const PAIR_BUDGET_CHARS = 24_000_000; // per file: characters the window search may read before it gives up (and reports) + +/** + * The text of the innermost `{ ... }` around `index` by naive brace counting (null when there is none in range) and the + * characters it cost. Reads at most PAIR_BACK_CHARS + PAIR_FORWARD_CHARS. Braces inside strings are counted, which + * a JSON string such as "a } b" can exploit; stringAwareWindow covers that, and both are searched. + */ +function braceWindow(input, index) { + // Native indexOf/lastIndexOf jump between braces, so a window without many braces costs a couple of memchr calls. + const from = Math.max(0, index - PAIR_BACK_CHARS); + const back = input.slice(from, index); + let open = -1; + let nearOpen = back.lastIndexOf('{'); + let nearClose = back.lastIndexOf('}'); + let depth = 0; + while (nearOpen !== -1) { + if (nearClose > nearOpen) { + depth += 1; + nearClose = nearClose === 0 ? -1 : back.lastIndexOf('}', nearClose - 1); + } else { + if (depth === 0) { + open = from + nearOpen; + break; + } + depth -= 1; + nearOpen = nearOpen === 0 ? -1 : back.lastIndexOf('{', nearOpen - 1); + } + } + if (open === -1) return { text: null, cost: 64 }; + const forward = input.slice(index, Math.min(input.length, index + PAIR_FORWARD_CHARS)); + let end = index + forward.length; + let nextOpen = forward.indexOf('{'); + let nextClose = forward.indexOf('}'); + depth = 0; + while (nextClose !== -1) { + if (nextOpen !== -1 && nextOpen < nextClose) { + depth += 1; + nextOpen = forward.indexOf('{', nextOpen + 1); + } else { + if (depth === 0) { + end = index + nextClose + 1; + break; + } + depth -= 1; + nextClose = forward.indexOf('}', nextClose + 1); + } + } + return { text: input.slice(open, end), start: open, cost: end - open }; +} + +/** + * Bracket tracking that knows about strings: a `}` or `)` inside "..." or '...' (backslash escapes honoured; a string + * never runs past a line break, so an apostrophe in a comment cannot swallow the file) does not close anything. + * `state` = { braces: [], parens: [] } holds the positions of the still-open brackets. With `stopAt` ('}' or ')'), the scan + * ends at the first such closer that closes nothing opened inside `text` and returns its index (otherwise -1). + */ +function scanBrackets(text, offset, state, stopAt) { + let quote = null; + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]; + if (quote !== null) { + if (ch === '\\') i += 1; + else if (ch === quote || ch === '\n') quote = null; + } else if (ch === '"' || ch === "'") { + quote = ch; + } else if (ch === '{') { + state.braces.push(offset + i); + } else if (ch === '(') { + state.parens.push(offset + i); + } else if (ch === '}') { + if (state.braces.length === 0 && stopAt === '}') return i; + state.braces.pop(); + } else if (ch === ')') { + if (state.parens.length === 0 && stopAt === ')') return i; + state.parens.pop(); + } + } + return -1; +} + +/** + * The string-aware innermost enclosing bracket pair around `index`: `{ ... }` for a JSON/JS/Go/HCL object, or `( ... )` + * for a call (create_var(name='X', value='Y')) when that is the innermost one. Returns { text, cost } (text null when + * there is no enclosing pair in range). Bounded to PAIR_BACK_CHARS + PAIR_FORWARD_CHARS characters. + */ +function stringAwareWindow(input, index) { + const from = Math.max(0, index - PAIR_BACK_CHARS); + const back = input.slice(from, index); + if (!back.includes('{') && !back.includes('(')) return { text: null, cost: 64 }; + const state = { braces: [], parens: [] }; + scanBrackets(back, from, state, null); + const brace = state.braces[state.braces.length - 1] ?? -1; + const paren = state.parens[state.parens.length - 1] ?? -1; + const open = Math.max(brace, paren); + if (open === -1) return { text: null, cost: index - from }; + const openChar = open === brace ? '{' : '('; + // Forward: brackets opened after the name field must be closed first; the first unmatched closer of our kind ends it. + const forward = input.slice(index, Math.min(input.length, index + PAIR_FORWARD_CHARS)); + const ahead = { braces: [], parens: [] }; + let end = index + forward.length; + const hit = scanBrackets(forward, index, ahead, openChar === '{' ? '}' : ')'); + if (hit !== -1) end = index + hit + 1; + return { text: input.slice(open, end), start: open, cost: index - from + (end - index) }; +} + +/** + * The lines of the YAML mapping (a list item, or a plain block) that the name line belongs to: the item's own first line + * and its sibling keys, in either direction, up to PAIR_YAML_LINES lines each way. An item ends at the next `- ` at + * or left of it, a dedent, or a document marker. Returns null when the key is not at the start of its line. + * Every search for a line break is bounded to PAIR_LINE_CHARS, so a huge single line costs a constant amount. + * Returns { text, lines } where `lines` are the [start offset in input, text offset] of each line, for locating a match. + */ +/** Join lines ({ text, start }) with "\n" into a window that remembers where each line came from. */ +function joinLines(lines) { + let offset = 0; + const parts = lines.map((line) => { + const part = { start: line.start, at: offset }; + offset += line.text.length + 1; + return part; + }); + return { text: lines.map((line) => line.text).join('\n'), parts }; +} + +/** The offset in the scanned input of `index` in a window's text (a window is one run of input, or a run per line). */ +function inputOffset(window, index) { + if (window.parts === undefined) return window.start + index; + let found = window.parts[0]; + for (const part of window.parts) { + if (part.at > index) break; + found = part; + } + return found.start + (index - found.at); +} + +function yamlBlockWindow(input, index) { + // Start of the line holding `at`, or -1 when the line is longer than PAIR_LINE_CHARS (not a YAML block line). + const startOfLine = (at) => { + const from = Math.max(0, at - PAIR_LINE_CHARS); + const found = input.slice(from, at).lastIndexOf('\n'); + return found !== -1 ? from + found + 1 : from === 0 ? 0 : -1; + }; + // The line starting at `start`: its text and where the next line starts (input.length + 1 when there is none in range). + const lineFrom = (start) => { + const chunk = input.slice(start, start + PAIR_LINE_CHARS + 1); + const newline = chunk.indexOf('\n'); + if (newline === -1) return { text: chunk.replace(/\r$/, ''), start, next: input.length + 1 }; + return { text: chunk.slice(0, newline).replace(/\r$/, ''), start, next: start + newline + 1 }; + }; + const indentOf = (text) => /^[ \t]*/.exec(text)[0].length; + const isDash = (text) => /^[ \t]*-(?:[ \t]|$)/.test(text); + const isMarker = (text) => /^(?:---|\.\.\.)[ \t]*$/.test(text); + + const lineStart = startOfLine(index); + if (lineStart === -1) return null; + const prefix = input.slice(lineStart, index); + if (!/^[ \t]*(?:-[ \t]+)*$/.test(prefix)) return null; + const keyCol = prefix.length; + const ownItem = prefix.includes('-'); + const own = lineFrom(lineStart); + const above = []; + if (!ownItem) { + let cursor = lineStart; + // Lines indented deeper than the key (the body of a block scalar such as `value: |` above the name) do not use up the + // PAIR_YAML_LINES allowance, so a long body cannot push its `value:` header out of the window; they have their own bound, + // and a blank line inside such a body (legal in a block scalar) does not end the walk. + let ordinary = 0; + let deep = 0; + let insideBody = false; + while (cursor > 0 && ordinary < PAIR_YAML_LINES && deep < MULTILINE_MAX_LINES) { + const previousStart = startOfLine(cursor - 1); + if (previousStart === -1) break; + const line = lineFrom(previousStart); + cursor = previousStart; + if (line.text.trim() === '') { + if (!insideBody) break; + deep += 1; + continue; + } + if (isMarker(line.text)) break; + if (isDash(line.text) && /^[ \t]*-[ \t]+/.exec(line.text)[0].length === keyCol) { + above.push(line); + break; + } + if (indentOf(line.text) < keyCol || (isDash(line.text) && indentOf(line.text) <= keyCol)) break; + above.push(line); + if (indentOf(line.text) > keyCol) { + insideBody = true; + deep += 1; + } else { + insideBody = false; + ordinary += 1; + } + } + } + const below = []; + let cursor = own.next; + for (let n = 0; n < PAIR_YAML_LINES && cursor <= input.length; n += 1) { + const line = lineFrom(cursor); + cursor = line.next; + if (line.text.trim() === '') continue; + if (isMarker(line.text)) break; + const indent = indentOf(line.text); + if (indent < keyCol || (isDash(line.text) && indent === keyCol && !ownItem)) break; + below.push(line); + } + return joinLines([...above.reverse(), own, ...below]); +} + +// The entry tags of XML/properties-style configuration: , , +// , V. +const XML_ENTRY_TAG = /<(?:add|setting|property|entry|item|param|parameter|variable|var|env|envvar|option|appsetting|pair|element|secret)(?![A-Za-z0-9_-])/gi; + +const XML_ENTRY_CLOSE = /<\/(?:add|setting|property|entry|item|param|parameter|variable|var|env|envvar|option|appsetting|pair|element|secret)[ \t\r\n]*>/gi; +const XML_ENTRY_CLOSE_ONCE = /<\/(?:add|setting|property|entry|item|param|parameter|variable|var|env|envvar|option|appsetting|pair|element|secret)[ \t\r\n]*>/i; +/** + * The XML entry around `index`: from the nearest opening entry tag before it (or the tag that holds it) to the end of that + * entry (`/>`, or its closing tag, or the start of the next entry), at most PAIR_XML_CHARS. Null when `index` is not in XML. + */ +function xmlWindow(input, index) { + const from = Math.max(0, index - PAIR_XML_CHARS); + const back = input.slice(from, index); + if (!back.includes('<')) return null; + let start = -1; + for (const tag of back.matchAll(XML_ENTRY_TAG)) start = from + tag.index; + if (start === -1) start = from + back.lastIndexOf('<'); + // An entry that already ended before `index` (a "/>" or a closing entry tag between its start and `index`) is a different + // element: the window starts at the first tag after that end, so its value= is never paired with a later name=. + const between = input.slice(start, index); + let ended = between.lastIndexOf('/>'); + if (ended !== -1) ended += 2; + for (const close of between.matchAll(XML_ENTRY_CLOSE)) ended = Math.max(ended, close.index + close[0].length); + if (ended > 0) { + const nextTag = between.indexOf('<', ended); + // No tag between the end and `index`: `index` is in text after the entry, nothing pairs with it. + start = nextTag === -1 ? index : start + nextTag; + } + const forward = input.slice(index, Math.min(input.length, index + PAIR_XML_CHARS)); + let end = forward.length; + const selfClose = forward.indexOf('/>'); + if (selfClose !== -1) end = Math.min(end, selfClose + 2); + const closing = XML_ENTRY_CLOSE_ONCE.exec(forward); + if (closing) end = Math.min(end, closing.index + closing[0].length); + XML_ENTRY_TAG.lastIndex = 0; + const next = XML_ENTRY_TAG.exec(forward); + if (next && next.index > 0) end = Math.min(end, next.index); + XML_ENTRY_TAG.lastIndex = 0; + return { text: input.slice(start, index + end), start }; +} + +// The value-carrying field of a name/value object. `valueFrom`, `values` and other longer names do not match. A YAML tag +// or anchor before the scalar (`!!str V`, `&a V`) is skipped. +const PAIR_VALUE_FIELD = + /(?V +const PAIR_XML_VALUE = /<(?:value|val|secret|content|data|default|string)(?:[ \t][^<>]{0,80})?>[ \t\r\n]*([^<>]{1,4096}?)[ \t\r\n]*<\//gi; + +// Where a value begins (the same field names as PAIR_VALUE_FIELD, whatever follows): read again from the input when it runs over +// several lines (a YAML block scalar, a quote closed on a later line, a folded scalar, a heredoc). +const PAIR_VALUE_START = + /(? with an optional chomping (+ -) and indentation (1-9) indicator, then only a comment. +const BLOCK_SCALAR_HEADER = /[|>](?:[-+][1-9]?|[1-9][-+]?)?(?=[ \t]*(?:#[^\n]*)?(?:\r?\n|$))/y; + +/** + * The body of a YAML block scalar whose header ends at `afterHeader`: the following lines indented deeper than the key (blank + * lines belong to it), as candidate values (each line, and all of them joined). Bounded like the other multi-line readers; a body + * that does not end within the bounds (or the file's budget) is reported as exhausted so the caller fails closed. + * @returns {{values: string[], end: number, exhausted: boolean, budget?: boolean}} + */ +function blockScalarValue(ctx, input, afterHeader, keyColumn) { + if (budgetSpent(ctx)) return exhaustedLiteral(ctx, input, afterHeader); + const lines = []; + let at = input.indexOf('\n', afterHeader); + let end = afterHeader; + let closed = at === -1; + let chars = 0; + at += 1; + for (let n = 0; !closed && n < MULTILINE_MAX_LINES && chars <= MULTILINE_MAX_CHARS; n += 1) { + if (at >= input.length) { + closed = true; + break; + } + const next = input.indexOf('\n', at); + const stop = next === -1 ? input.length : next; + const raw = input.slice(at, Math.min(stop, at + 4096)).replace(/\r$/, ''); + chars += stop - at + 1; + if (raw.trim() !== '') { + if (/^[ \t]*/.exec(raw)[0].length <= keyColumn) { + closed = true; + break; + } + lines.push(raw.trim()); + end = stop; + } + if (next === -1) { + closed = true; + break; + } + at = next + 1; + } + if (!closed) return exhaustedLiteral(ctx, input, afterHeader); + if (!spendMultiline(ctx, end - afterHeader + 1)) return exhaustedLiteral(ctx, input, afterHeader); + return { values: bodyValues(lines.join('\n')), end, exhausted: false }; +} + +/** The lines of a JSON string value whose line breaks are written as \n escapes, as candidate values (each line, and all joined). */ +function escapedLineValues(raw) { + if (!/\\[nr]/.test(raw)) return []; + const lines = raw.split(/\\r\\n|\\n|\\r/).map((line) => unescapeQuoted(line).trim()).filter((line) => line !== ''); + return lines.length > 1 ? [...lines, lines.join(' ')] : []; +} + +/** + * Value fields of the window whose value is not on the field's line: a block scalar (`value: |`, `>-`, `|2`, after a tag such as + * `!Sub`), a quote closed on a later line, a heredoc or triple-quoted string, a scalar folded over indented lines. The value is + * read from the input (the window only says where it starts) and judged like a quoted value. Returns a hit ({ abs }), an + * unverifiable read ({ exhausted }) or null. + */ +function multilineFieldSecret(window, kind, ctx, input, read) { + for (const start of window.text.matchAll(PAIR_VALUE_START)) { + const keyAt = inputOffset(window, start.index); + const valueAt = inputOffset(window, start.index + start[0].length); + if (read.has(valueAt)) continue; + read.add(valueAt); + const keyColumn = keyAt - ctx.lineStart(keyAt); + BLOCK_SCALAR_HEADER.lastIndex = valueAt; + const header = BLOCK_SCALAR_HEADER.exec(input); + const body = + header !== null + ? blockScalarValue(ctx, input, valueAt + header[0].length, keyColumn) + : (multilineValue(ctx, input, valueAt) ?? continuedValue(ctx, input, keyAt, valueAt)); + if (body === null) continue; + if (body.exhausted) return { exhausted: true, budget: body.budget === true }; + const judged = body.values.some((value) => + isSecretValue({ kind, value, quoted: true, separator: ':', mode: ctx.mode, minLength: ctx.minStrong, catalog: ctx.catalog }), + ); + if (judged) return { abs: { start: valueAt, end: body.end } }; + } + return null; +} + +/** + * Is any value field (either order) in the window text a non-placeholder secret? Returns where it sits ({ index, length } in the + * window text, or { abs } for a value read from the input), { exhausted } when a multi-line value cannot be verified, or null. + */ +function windowHoldsSecretValue(window, kind, ctx, escaped, input, read = new Set()) { + const text = window.text; + const unescaped = escaped ? text.replace(/\\(["'])/g, '$1') : text; + // Where the offending value sits (offset and length in the window's text). An escaped window changes its length, so the + // whole window stands for it then. + const at = (field) => (escaped ? { index: 0, length: text.length } : { index: field.index, length: field[0].length }); + const judge = (value, quoted) => + isSecretValue({ kind, value, quoted, separator: ':', mode: ctx.mode, minLength: ctx.minStrong, catalog: ctx.catalog }); + for (const field of unescaped.matchAll(PAIR_VALUE_FIELD)) { + const quotedValue = field[2] ?? field[3]; + const quoted = quotedValue !== undefined; + if (quoted ? judge(unescapeQuoted(quotedValue), true) || escapedLineValues(quotedValue).some((line) => judge(line, true)) : judge(field[4], false)) { + return at(field); + } + } + if (unescaped.includes(' judge(line, true))) return at(field); + } + } + if (!escaped) return multilineFieldSecret(window, kind, ctx, input, read); + return null; +} + +/** + * Record on the match where the VALUE field sits (the name field is the match itself), which can be lines away from the + * name. --history and --range report a finding when the commit added a line of the match or a line of the value, not a + * line in between (the reported line and `spanEnd` are untouched, so the tree report and the inline allow marker behave as before). + */ +function attributeTo(m, window, found, coarse = false) { + if (found.abs !== undefined) { + // A value read straight from the input (a block scalar, a multi-line literal): the marker line through its last line. + m.attrStart = found.abs.start; + m.attrEnd = Math.max(found.abs.end, found.abs.start + 1); + m.attrCoarse = false; + return; + } + const first = inputOffset(window, found.index); + const last = inputOffset(window, found.index + Math.max(found.length - 1, 0)); + m.attrStart = first; + m.attrEnd = last + 1; + // A coarse range (a whole escaped window, the whole file) does not say which line holds the value, so an allow marker + // on a line of it must not suppress the finding. + m.attrCoarse = coarse; +} + +/** The window search ran out of budget: the finding rests on the whole file (a history scan blames any added line). */ +function attributeToFile(m) { + m.attrStart = 0; + m.attrEnd = m.input.length; + m.attrCoarse = true; +} + +/** + * Judge a name/value object whose secret-like name field is at `m`: is any value field in the same bounded JSON/HCL/YAML/ + * XML object or call (either order, other fields in between) a non-placeholder secret? Reads are budgeted per file; + * running out of budget reports the file once (hostile input must not be silently skipped). + */ +function pairValueIsSecret(m, kind, ctx) { + if (ctx.pairExhausted) return false; // already reported for this file, and nothing more is read + ctx.pairBudget ??= PAIR_BUDGET_CHARS; + const windows = []; // { text, start } for one run of the input, { text, parts } for a run per line + const brace = braceWindow(m.input, m.index); + ctx.pairBudget -= brace.cost; + if (brace.text !== null) windows.push(brace); + const aware = stringAwareWindow(m.input, m.index); + ctx.pairBudget -= aware.cost; + if (aware.text !== null && aware.text !== brace.text) windows.push(aware); + const yaml = yamlBlockWindow(m.input, m.index); + if (yaml !== null) { + ctx.pairBudget -= yaml.text.length; + windows.push(yaml); + } + const xml = xmlWindow(m.input, m.index); + if (xml !== null) { + ctx.pairBudget -= xml.text.length; + windows.push(xml); + } + if (ctx.pairBudget < 0) { + // Out of budget: the file is too dense with secret-like names to verify. Report it (once: one finding fails the run). + ctx.pairExhausted = true; + attributeToFile(m); + return true; + } + const escaped = m[1].startsWith('\\'); // a JSON document stored as a string: {\"key\":\"X\",\"value\":\"Y\"} + const read = new Set(); // the same value is found through several windows: read it once + for (const window of windows) { + const found = windowHoldsSecretValue(window, kind, ctx, escaped, m.input, read); + if (found !== null) return pairHit(m, window, found, ctx, escaped); + } + return false; +} + +/** A window search found a value (or could not verify one): record where, and report the pair. */ +function pairHit(m, window, found, ctx, coarse = false) { + if (found.exhausted !== true) { + attributeTo(m, window, found, coarse); + return true; + } + // A multi-line value that does not end within the bounds cannot be verified: reported (fail closed), once per file when the + // file's budget is what ran out. + attributeToFile(m); + if (!found.budget) return true; + if (ctx.multilineReported) return false; + ctx.multilineReported = true; + return true; +} + +/** Value fields inside the child block or object that belongs to a name: `NAME:` followed by indented lines, or `NAME: {`. */ +function childHoldsSecretValue(m, kind, ctx, keyIndent) { + if (ctx.pairExhausted) return false; + ctx.pairBudget ??= PAIR_BUDGET_CHARS; + const input = m.input; + const after = m.index + m[0].length; + let window; + if (input[after] === '{') { + const forward = input.slice(after + 1, Math.min(input.length, after + 1 + PAIR_FORWARD_CHARS)); + const state = { braces: [], parens: [] }; + const hit = scanBrackets(forward, after + 1, state, '}'); + window = { text: forward.slice(0, hit === -1 ? forward.length : hit), start: after + 1 }; + } else { + const lines = []; + // `after` sits at the end of the name line (or a comment before its line break): the children are the following, + // more indented lines. Each line is cut at PAIR_LINE_CHARS, so a huge line costs a constant amount. + let lineBreak = input.indexOf('\n', after); + let childIndent = -1; // only the direct children count: `secrets:` is not the name of a value nested two levels down + for (let n = 0; n < PAIR_YAML_LINES * 2 && lineBreak !== -1 && lines.length < PAIR_YAML_LINES; n += 1) { + const from = lineBreak + 1; + const next = input.indexOf('\n', from); + const line = input.slice(from, next === -1 ? Math.min(input.length, from + PAIR_LINE_CHARS) : Math.min(next, from + PAIR_LINE_CHARS)).replace(/\r$/, ''); + lineBreak = next; + if (line.trim() === '') continue; + const indent = /^[ \t]*/.exec(line)[0].length; + if (/^(?:---|\.\.\.)[ \t]*$/.test(line) || indent <= keyIndent) break; + if (childIndent === -1) childIndent = indent; + if (indent === childIndent) lines.push({ text: line, start: from }); + } + window = joinLines(lines); + } + ctx.pairBudget -= window.text.length + 64; + if (ctx.pairBudget < 0) { + ctx.pairExhausted = true; + attributeToFile(m); + return true; + } + const found = windowHoldsSecretValue(window, kind, ctx, false, input); + return found !== null && pairHit(m, window, found, ctx); +} + +// --------------------------------------------------------------------------- +// Secrets passed on a command line: gh secret set NAME --body V, netlify env:set NAME V, aws ssm put-parameter ... +// --------------------------------------------------------------------------- + +// Flags that carry the variable name, and flags that carry its value, in the CLIs above. +const CLI_NAME_FLAGS = new Set(['--name', '--key', '--secret-name', '--parameter-name']); +const CLI_VALUE_FLAGS = new Set(['--body', '-b', '--value', '--secret-string', '--string-value', '--secret-value', '--plaintext']); +// Flags that take a separate argument but are neither of the above (so it is not mistaken for a name or a value). +const CLI_OTHER_ARG_FLAGS = new Set([ + '--type', '--description', '--env', '--environment', '--app', '-a', '--repo', '-R', '--org', '-o', '--vault-name', '--region', + '--profile', '--scope', '--context', '--site', '-s', '--project', '--secret-id', '--tags', '--kms-key-id', '--key-id', + '--visibility', '--repos', '--user', '--env-file', '--config', '--namespace', '-n', '--from-file', '--tier', '--data-type', +]); + +/** Shell words of one command line: quotes honoured, stops at an unquoted pipe, ;, &, > or comment. `<<<` is a word of its own. */ +function shellWords(line) { + const words = []; + let word = null; + let quote = null; + let ansi = false; // inside $'...': backslash escapes work there + const push = () => { + if (word !== null) words.push(word); + word = null; + }; + for (let i = 0; i < line.length; i += 1) { + const ch = line[i]; + if (quote !== null) { + if (ch === '\\' && (quote === '"' || ansi) && i + 1 < line.length) { + word += line[i + 1]; + i += 1; + } else if (ch === quote) { + quote = null; + ansi = false; + } else word += ch; + } else if (ch === '$' && (line[i + 1] === "'" || line[i + 1] === '"')) { + quote = line[i + 1]; // $'...' (ANSI-C) and $"..." (locale) quote like '...' and "..."; the $ is not part of the word + ansi = quote === "'"; + word ??= ''; + i += 1; + } else if (ch === '"' || ch === "'") { + quote = ch; + word ??= ''; + } else if (ch === ' ' || ch === '\t' || ch === '\r') { + push(); + } else if (ch === '|' || ch === ';' || ch === '&' || ch === '>' || (ch === '#' && word === null)) { + break; + } else if (ch === '<' && line.startsWith('<<<', i)) { + push(); + words.push('<<<'); + i += 2; + } else if (ch === '\\' && i + 1 < line.length) { + word = (word ?? '') + line[i + 1]; + i += 1; + } else { + word = (word ?? '') + ch; + } + } + push(); + words.open = quote !== null; // the line ended inside a quote: the last word is cut off (usually a string in surrounding code) + return words; +} + +/** + * The (name, value) pairs of one CLI invocation. `verb` names the tool (see CLI_TOOLS): `positionalValue` tools take + * `NAME VALUE`; every tool takes --name/--value style flags, --from-literal=K=V, a `<<<` here-string or an + * `echo V |` pipe for its first positional name. `K=V` positionals are pairs only where `assignments` is true. + */ +function cliPairs(words, { positionalValue, assignments }, piped) { + const pairs = []; + const positional = []; + let flagName = null; + let flagValue = null; + let hereString = null; + for (let i = 0; i < words.length; i += 1) { + const word = words[i]; + if (word === '<<<') { + hereString = words[i + 1] ?? null; + i += 1; + } else if (word.startsWith('<<<') && word.length > 3) { + hereString = word.slice(3); + } else if (word.startsWith('<<')) { + if (word === '<<' || word === '<<-') i += 1; // a heredoc delimiter, not a positional; its body is read separately + } else if (word === '<') { + i += 1; // a redirect from a file + } else if (word.startsWith('--from-literal=')) { + const eq = word.indexOf('=', 15); + if (eq !== -1) pairs.push([word.slice(15, eq), word.slice(eq + 1)]); + } else if (word === '--from-literal') { + const literal = words[i + 1] ?? ''; + const eq = literal.indexOf('='); + if (eq !== -1) pairs.push([literal.slice(0, eq), literal.slice(eq + 1)]); + i += 1; + } else if (word.startsWith('-')) { + const eq = word.indexOf('='); + const flag = eq === -1 ? word : word.slice(0, eq); + const inline = eq === -1 ? null : word.slice(eq + 1); + const take = () => (inline !== null ? inline : ((i += 1), words[i] ?? null)); + if (CLI_NAME_FLAGS.has(flag)) flagName = take(); + else if (CLI_VALUE_FLAGS.has(flag)) flagValue = take(); + else if (CLI_OTHER_ARG_FLAGS.has(flag)) take(); + } else if (assignments && /^[A-Za-z_][A-Za-z0-9_]*=/.test(word)) { + pairs.push([word.slice(0, word.indexOf('=')), word.slice(word.indexOf('=') + 1)]); + } else { + positional.push(word); + } + } + const name = flagName ?? positional[0] ?? null; + const value = flagValue ?? hereString ?? (positionalValue ? positional[1] : null) ?? piped; + if (name !== null && value !== null && value !== undefined && !name.includes('=')) pairs.push([name, value]); + return pairs; +} + +// A pipeline stage between the source of a value and the CLI that leaves the text as it is (or close enough to still be the secret). +const PIPE_FILTER = /^(?:cat|tee|tr|head|tail|sed|awk|cut|base64|paste|rev|iconv)(?![A-Za-z0-9_-])/; + +/** Index of the first unquoted single `|` in `text`, or -1 when a `;`, `&`, `||` or the end comes first. */ +function firstPipe(text) { + let quote = null; + for (let i = 0; i < text.length; i += 1) { + const ch = text[i]; + if (quote !== null) { + if (ch === '\\' && quote === '"') i += 1; + else if (ch === quote) quote = null; + } else if (ch === '"' || ch === "'") quote = ch; + else if (ch === '\\') i += 1; + else if (ch === '|') return text[i + 1] === '|' ? -1 : i; + else if (ch === ';' || ch === '&') return -1; + } + return -1; +} + +/** + * Is the text after a pipe a run of value-preserving filters followed by the stage that holds the CLI (a wrapper such as + * `sudo`, `env A=1`, `time`, `sh -c "`)? A stage that chains another command (`;`, `&&`, `||`) is not. + */ +function pipeTailReachesCli(tail) { + const stages = tail.split('|'); + const last = stages.pop(); + return stages.every((stage) => PIPE_FILTER.test(stage.trim())) && /^[^;&<>]{0,200}$/.test(last); +} + +/** + * The body of a heredoc that starts at `from`, up to the line that is its delimiter. Reads at most MULTILINE_MAX_LINES lines / + * MULTILINE_MAX_CHARS characters (and the file's multi-line budget). `closed` is false when the delimiter is not found within + * those bounds: the body cannot be verified, so the caller fails closed. `end` is the index where the read stopped. + * @returns {{lines: string[], end: number, closed: boolean}} + */ +function heredocBody(ctx, input, from, word) { + if (budgetSpent(ctx)) return { lines: [], end: from, closed: false }; // nothing left to read with: unverified + const lines = []; + let at = from; + let closed = false; + let end = from; + for (let n = 0; n < MULTILINE_MAX_LINES && at <= input.length && at - from <= MULTILINE_MAX_CHARS; n += 1) { + const lineEnd = input.indexOf('\n', at); + const stop = lineEnd === -1 ? input.length : lineEnd; + const line = input.slice(at, Math.min(stop, at + 4096)).replace(/\r$/, ''); + end = stop; + if (line.trim() === word) { + closed = true; + break; + } + lines.push(line); + if (lineEnd === -1) break; + at = lineEnd + 1; + } + if (!spendMultiline(ctx, end - from)) closed = false; + return { lines, end, closed }; +} + +/** + * The values a CLI match receives on standard input, from the same command line: `echo V | cli`, `printf '%s' V | cli` + * (also mid-line after any `;`, `&&`, `||`, `|`, `(`, `then`, `do`, `sudo`, `env A=1`, a `bash -c "` quote or a YAML `run:`; + * the source only has to be the word `echo`/`printf` before the pipe, so indentation, tabs, CRLF and list prefixes do not + * matter), through value-preserving filters (`| tr -d '\n' |`), across a `\` or `|` line continuation, and heredocs + * (`cat < 0 && !before.includes('|'); hops += 1) { + const prevEnd = start - 1; + const prevStart = input.lastIndexOf('\n', prevEnd - 1) + 1; + const previous = input.slice(Math.max(prevStart, prevEnd - 400), prevEnd).replace(/\r$/, ''); + if (!/(?:\\|\|)[ \t]*$/.test(previous)) break; + before = `${previous.replace(/\\[ \t]*$/, '')} ${before}`; + start = prevStart; + } + const values = []; + const heredoc = { end: 0, unterminated: false }; + const bodyStart = ctx.lineEnd(m.index) + 1; + const readHeredoc = (word) => { + const body = heredocBody(ctx, input, bodyStart, word); + heredoc.end = Math.max(heredoc.end, body.end); + if (!body.closed) heredoc.unterminated = true; + values.push(...bodyValues(body.lines.join('\n'))); + }; + // echo / printf: the last usable source before the pipe. + const sources = [...before.matchAll(/(?= 0 && k >= sources.length - 4 && values.length === 0; k -= 1) { + const rest = before.slice(sources[k].index + sources[k][0].length); + const pipe = firstPipe(rest); + if (pipe === -1 || !pipeTailReachesCli(rest.slice(pipe + 1))) continue; + const words = shellWords(rest.slice(0, pipe)); + while (words.length > 0 && /^(?:-[A-Za-z]+|--)$/.test(words[0])) words.shift(); + if (words.length === 0) continue; + if (sources[k][1] === 'echo') values.push(words.join(' ')); + else if (/%[-+ #0-9.]*[sbqdi]/.test(words[0])) values.push(...words.slice(1)); // printf FORMAT ARGS: the arguments + else values.push(words[0].replace(/(?:\\[nr])+$/, '')); // printf 'V\n' + } + // Heredocs: the body starts on the line after the one holding the CLI. + const opener = /(? { + const value = stripQuotes(String(raw ?? '').replace(/\r$/, '').trim()); + return value.length >= CREDENTIAL_FILE_MIN_LENGTH && !isPlaceholder(value); +}; + +// netrc tokens are separated by blanks or line breaks; scripts write them with printf and a literal \n. +const NETRC_SEP = String.raw`(?:[ \t\r\n]|\\[nr]){1,64}`; + +/** The anonymous-FTP convention (`password guest@`, `anonymous`): not a secret. */ +const isAnonymousFtpValue = (value) => /@$/.test(value) || /^(?:anonymous|guest)$/i.test(value); + +/** True when the match sits on a line whose first non-blank character is `#` (a comment in credential files). */ +function onCommentLine(m, ctx) { + const start = ctx.lineStart(m.index); + return /^[ \t]*#/.test(m.input.slice(start, m.index + 1)); +} + +// --------------------------------------------------------------------------- +// Command-style assignments: shell syntaxes that set a variable WITHOUT "name=value" +// fish set -gx NAME value [value ...] csh/tcsh setenv NAME value Windows setx NAME value, set "NAME=value" +// sh export NAME value PowerShell $env:NAME = 'v', [Environment]::SetEnvironmentVariable('NAME','v'), +// fish universal variables file SETUVAR --export NAME:value Set-Item Env:NAME 'v' +// Judged like every other assignment (same name kinds, placeholder and passphrase logic), in shell-like contexts only: +// shell script and config files, fenced blocks of Markdown, and the distinctive forms (a flagged fish `set`, setenv, setx, +// PowerShell) anywhere, so a script written by heredoc into a YAML step or a Python string is read too. +// --------------------------------------------------------------------------- + +const SHELL_SCRIPT_EXTENSIONS = new Set(['.sh', '.bash', '.zsh', '.fish', '.ksh', '.csh', '.tcsh', '.bat', '.cmd', '.ps1', '.psm1', '.nu', '.envrc']); +const SHELL_FENCE_LANGS = /^(?:|sh|bash|zsh|fish|ksh|csh|tcsh|shell|shellscript|shell-session|console|terminal|bat|batch|cmd|powershell|pwsh|ps1|posh|nu|nushell|dockerfile|docker|yaml|yml|env|dotenv|ini|toml|makefile|make|text|txt|plaintext)$/i; +const SHELL_DOTFILE = /^\.(?:bash|zsh|ksh|csh|tcsh)?[a-z_]*(?:rc|profile|login|zshenv|aliases)$/; +const isShellScriptPath = (filePath) => { + const base = path.posix.basename(filePath.split(path.sep).join('/')).toLowerCase(); + return SHELL_SCRIPT_EXTENSIONS.has(path.posix.extname(base)) || SHELL_DOTFILE.test(base); +}; + +// What may stand before a command on its line: nothing, a list dash, a run-like YAML key, a prompt, RUN, `@` (batch), an +// opening quote; or a separator / `-c` / then / do when the command follows another one. +const COMMAND_AT_LINE_START = /^[ \t]*(?:[-*][ \t]+)?(?:(?:run|script|command|cmd|entrypoint|shell|args):[ \t]+)?(?:RUN[ \t]+|[$>%][ \t]+|@)?["']?$/i; +const COMMAND_AFTER_SEPARATOR = /(?:[;&|({"'`]|\b(?:then|do|else|and|or|begin)|[ \t]-c|--command)[ \t]*["']?$/; +// `export A B` exports the variables A and B: words shaped like variable names (UPPER_SNAKE or lower_snake) are names, not a value. +const SHELL_VARIABLE_NAME = /^(?:[A-Z_][A-Z0-9_]*|[a-z_][a-z0-9_]*)$/; +const SHELL_REFERENCE = /^(?:\$[A-Za-z_{(@*#?0-9]|%[^%\s]{1,100}%$|![^!\s]{1,100}!$)/; + +/** The blank-separated words of the rest of one line, quotes honoured (see the class comment above). At most 32 words, 4096 characters. */ +function shellTail(input, start) { + let end = input.indexOf('\n', start); + if (end === -1 || end - start > 4096) end = Math.min(input.length, start + 4096); + const line = input.slice(start, end).replace(/\r$/, ''); + const words = []; + let text = null; + let quoted = false; + let expr = false; // the word holds a command substitution or a group: a reference, not a literal + let quote = null; + let openLength = 0; + let depth = 0; + let tick = false; + const push = () => { + if (text !== null) words.push({ text, quoted, expr: expr || text.includes('$(') }); + text = null; + quoted = false; + expr = false; + }; + let i = 0; + for (; i < line.length && words.length < 32; i += 1) { + const ch = line[i]; + if (quote !== null) { + if (ch === '\\' && quote === '"' && i + 1 < line.length) { + text += line[i + 1]; + i += 1; + } else if (ch === quote) quote = null; + else text += ch; + } else if (tick) { + if (ch === '`') tick = false; + text += ch; + } else if (depth > 0) { + if (ch === '(') depth += 1; + else if (ch === ')') depth -= 1; + text += ch; + } else if (ch === '"' || ch === "'") { + quote = ch; + openLength = (text ?? '').length; + text ??= ''; + quoted = true; + } else if (ch === '`') { + tick = true; + expr = true; + text = (text ?? '') + ch; + } else if (ch === '(') { + depth = 1; + expr = true; + text = (text ?? '') + ch; + } else if (ch === ' ' || ch === '\t') push(); + else if (ch === ';' || ch === '&' || ch === '|' || ch === '>' || ch === '<' || (ch === '#' && text === null)) break; + else if (ch === '\\' && i + 1 < line.length) { + text = (text ?? '') + line[i + 1]; + i += 1; + } else text = (text ?? '') + ch; + } + // A quote that never closes belongs to the string around the command (`fish -c "set -gx NAME 'v'"`): what it opened is dropped. + if (quote !== null) { + text = text.slice(0, openLength); + quoted = false; + if (text === '') text = null; + } + push(); + return words; +} + +/** Is any word (or all of them together, as one passphrase) a secret value for a name of this kind? */ +function shellWordsAreSecret(kind, words, ctx) { + const literal = words.filter( + (w) => !w.expr && !SHELL_REFERENCE.test(w.text) && !/^\(.*\)$/.test(w.text) && !(!w.quoted && (w.text === '=' || /^\/[A-Za-z]$/.test(w.text))), + ); + const judge = (value, quoted) => + isSecretValue({ kind, value, quoted, separator: '=', mode: 'config', minLength: ctx.minStrong, catalog: ctx.catalog }); + if (literal.some((w) => (w.text !== '' || w.quoted) && judge(w.text, w.quoted))) return true; + // fish: `set -gx NAME correct horse battery staple` is a list, and a passphrase written without quotes. + return literal.length > 1 && literal.length === words.length && judge(literal.map((w) => w.text).join(' '), true); +} + +/** Is the command at `index` in a place a command can start (see COMMAND_AT_LINE_START)? */ +function startsCommand(m, ctx) { + const lineStart = ctx.lineStart(m.index); + if (m.index - lineStart > 400) return COMMAND_AFTER_SEPARATOR.test(m.input.slice(m.index - 400, m.index)); + const before = m.input.slice(lineStart, m.index); + return COMMAND_AT_LINE_START.test(before) || COMMAND_AFTER_SEPARATOR.test(before); +} + +/** + * May a command-style assignment at this match be judged? `distinctive` syntax (a fish set with flags, setenv, setx, the + * PowerShell forms) is judged in every kind of file except plain prose, where only fenced blocks count. The bare + * `set NAME value` / `export NAME value` also need a shell-like file, a fenced shell block, or a run-like line. + */ +function commandContextOk(m, ctx, distinctive) { + if (ctx.mode === 'prose') return ctx.fenceLang(m.index) !== undefined && SHELL_FENCE_LANGS.test(ctx.fenceLang(m.index)); + if (!startsCommand(m, ctx)) return false; + if (distinctive) return true; + if (isShellScriptPath(ctx.path)) return true; + const before = m.input.slice(Math.max(ctx.lineStart(m.index), m.index - 400), m.index); + return /(?:^[ \t]*(?:[-*][ \t]+)?(?:run|script|command|cmd|entrypoint|shell|args):[ \t]+|^[ \t]*RUN[ \t]+|[$>][ \t]+|[;&|({][ \t]*|\b(?:then|do)[ \t]+|[ \t]-c[ \t]+)["']?$/i.test(before); +} + +/** The value of a PowerShell string literal starting at `start` ('...' doubles its quote; "..." escapes with a backtick), or null. */ +function powershellString(input, start) { + const quote = input[start]; + if (quote !== "'" && quote !== '"') return null; + let out = ''; + for (let i = start + 1; i < input.length && i - start < 4098; i += 1) { + const ch = input[i]; + if (ch === '\n') return null; + if (ch === quote) { + if (input[i + 1] === quote) { + out += quote; + i += 1; + } else return { text: out, end: i + 1, expandable: quote === '"' }; + } else if (ch === '`' && quote === '"' && i + 1 < input.length) { + out += input[i + 1]; + i += 1; + } else out += ch; + } + return null; +} + +/** A PowerShell literal is a candidate value unless it is a variable or a sub-expression ("$env:HOME", "$($x.Token)"). */ +function powershellValueIsSecret(kind, literal, ctx) { + if (literal === null || SHELL_REFERENCE.test(literal.text) || literal.text.includes('$(')) return false; + return isSecretValue({ kind, value: literal.text, quoted: true, separator: '=', mode: 'config', minLength: ctx.minStrong, catalog: ctx.catalog }); +} + +// Function-call forms that set an environment variable or a system property to a literal: +// C# / .NET Environment.SetEnvironmentVariable("NAME", "v"[, EnvironmentVariableTarget.Machine]) Win32 SetEnvironmentVariableW(L"NAME", L"v") +// Go os.Setenv("NAME", "v") Python os.putenv("NAME", "v"), os.environ.setdefault("NAME", "v") +// Java System.setProperty("name", "v"), props.setProperty("name", "v"), System.getenv().put("NAME", "v"), pb.environment().put(...) +// Ruby ENV.store("NAME", "v") Deno Deno.env.set("NAME", "v") Rust std::env::set_var("NAME", "v") +// C / PHP / Swift / Lua setenv("NAME", "v", 1), putenv("NAME=v"), _putenv_s("NAME", "v"), g_setenv(...), apache_setenv(...) +// Elixir System.put_env("NAME", "v") Erlang os:putenv("NAME", "v") +// (`ENV["NAME"] = "v"`, `os.environ["NAME"] = "v"`, `process.env.NAME = "v"`, `$_ENV['NAME'] = 'v'` and `$env:NAME = 'v'` are assignments.) +// Every alternative is a fixed call name, so a match starts at a name and reads a bounded argument list. +const ENV_SETTER_CALLEE = [ + String.raw`(?:System\.)?Environment\.SetEnvironmentVariable`, + String.raw`(?:os|syscall)\.Setenv`, + String.raw`os\.(?:putenv|environ\.setdefault)`, + String.raw`System\.(?:setProperty|getProperties\(\)\.setProperty|getenv\(\)\.put|put_env)`, + String.raw`[A-Za-z_$][A-Za-z0-9_$]{0,64}\.setProperty`, + String.raw`(?:environment|getenv)\(\)\.put`, + String.raw`ENV\.store`, + String.raw`Deno\.env\.set`, + String.raw`(?:std::)?env::set_var`, + String.raw`os:putenv`, + String.raw`(?:g_|_)?(?:setenv|putenv)(?:_s)?`, + String.raw`SetEnvironmentVariable[AW]?`, + String.raw`apache_setenv`, +].join('|'); +// The name, then either `, value` (two arguments) or `=value` inside the same string (putenv). Group 1 = the quote of the name, 2 = the name, +// 3 = "=" for the one-string form. +const ENV_SETTER_CALL = new RegExp( + String.raw`(? 4098 || input.slice(i + 3, close).includes('\n')) return null; + return { text: input.slice(i + 3, close), end: close + 3, interpolated }; + } + let out = ''; + for (let j = i + 1; j < input.length && j - i < 4098; j += 1) { + const ch = input[j]; + if (ch === '\n') return null; + if (ch === quote) { + if (verbatim && input[j + 1] === quote) { + out += quote; + j += 1; + } else return { text: out, end: j + 1, interpolated }; + } else if (ch === '\\' && !raw && !verbatim && j + 1 < input.length) { + out += input[j + 1]; + j += 1; + } else out += ch; + } + return null; +} + +/** fish `set` flags that read or remove a variable instead of assigning it (-e -q -S -n -h and their long forms). */ +function fishFlagsAssign(flags) { + return !flags.split(/[ \t]+/).some((flag) => (flag.startsWith('--') ? /^--(?:erase|query|show|names|help|list)$/.test(flag) : /^-[A-Za-z]*[eqSnh]/.test(flag))); +} + +// --------------------------------------------------------------------------- +// Values that span lines +// TOML """...""" and '''...''' (also Python, Kotlin, Java text blocks), HCL / Ruby / Perl / PHP heredocs (< line.trim()).filter((line) => line !== ''); + return lines.length > 1 ? [...lines, lines.join(' ')] : lines; +} + +/** Charge `n` characters against the file's multi-line budget. False once it is used up. */ +function spendMultiline(ctx, n) { + ctx.multilineBudget = (ctx.multilineBudget ?? MULTILINE_BUDGET_CHARS) - n; + return ctx.multilineBudget >= 0; +} + +/** The literal does not end within the bounds (or the file's budget is gone: `budget`): it cannot be verified, so it is reported. */ +function exhaustedLiteral(ctx, input, from) { + const budget = !spendMultiline(ctx, MULTILINE_MAX_CHARS); + return { values: [], end: Math.min(input.length, from + MULTILINE_MAX_CHARS), exhausted: true, budget }; +} +const budgetSpent = (ctx) => (ctx.multilineBudget ?? MULTILINE_BUDGET_CHARS) < 0; + +/** + * The verdict for a multi-line read that ran out of a bound: reported at the first such literal of a file, and once the file's + * whole budget is gone, once (the run fails on one finding; nothing more is read). + */ +function exhaustedVerdict(m, ctx, read) { + if (!read.budget) return true; + if (ctx.multilineReported) return false; + ctx.multilineReported = true; + attributeToFile(m); + return true; +} + +/** + * Index of the closing `quote` of a string body that starts at `from`, or -1 when it does not close within the bounds. + * `escapes`: a backslash makes the next character part of the string (so an escaped quote does not close it). + * `pair`: a doubled quote is one quote of the content (YAML 'it''s'). `tripled`: the closing delimiter is three quotes. + * `stop`: do not look at or beyond this index. + */ +function closingQuote(input, from, quote, { escapes, pair = false, tripled = false, stop = input.length }) { + const limit = Math.min(stop, from + MULTILINE_MAX_CHARS); + let lines = 0; + for (let i = from; i < limit; i += 1) { + const ch = input[i]; + if (ch === '\n') { + lines += 1; + if (lines > MULTILINE_MAX_LINES) return -1; + } else if (ch === '\\' && escapes) i += 1; + else if (ch === quote) { + if (tripled) { + if (input[i + 1] === quote && input[i + 2] === quote) return i; + } else if (pair && input[i + 1] === quote) i += 1; + else return i; + } + } + return -1; +} + +/** Undo the escapes of a multi-line basic string: a line-ending backslash joins the lines, and \x is x. */ +const unescapeMultiline = (text) => text.replace(/\\\r?\n[ \t\r\n]*/g, '').replace(/\\(.)/g, '$1'); + +/** + * A literal that starts at `valueStart` and runs over more than one line (or is written in a form the one-line value reader + * cannot see), or null when the value is an ordinary one-line value. + * @returns {{values: string[], end: number, exhausted: boolean} | null} + */ +function multilineValue(ctx, input, valueStart) { + const first = input[valueStart]; + // TOML / Python / Kotlin triple quotes, on one line or several. + TRIPLE_QUOTE.lastIndex = valueStart; + const triple = TRIPLE_QUOTE.exec(input); + if (triple !== null) { + if (budgetSpent(ctx)) return exhaustedLiteral(ctx, input, valueStart); + const quote = triple[1]; + const bodyStart = valueStart + triple[0].length; + const literalToml = quote === "'''" && /\.toml$/i.test(ctx.path); + const close = closingQuote(input, bodyStart, quote[0], { escapes: !literalToml, tripled: true }); + if (close === -1 || !spendMultiline(ctx, close - bodyStart)) return exhaustedLiteral(ctx, input, bodyStart); + const raw = input.slice(bodyStart, close).replace(/^\r?\n/, ''); + const isRawString = /[rR]/.test(triple[0].slice(0, -3)); + return { values: bodyValues(quote === '"""' && !isRawString ? unescapeMultiline(raw) : raw), end: close + 2, exhausted: false }; + } + // Heredocs: HCL <= input.length || next - lineEnd > MULTILINE_MAX_CHARS) break; + at = next + 1; + } + return exhaustedLiteral(ctx, input, lineEnd); + } + } + // PowerShell here-string: @' ... '@ and @" ... "@ (the closing mark starts a line). + if (first === '@') { + HERE_STRING_OPENER.lastIndex = valueStart; + const opener = HERE_STRING_OPENER.exec(input); + if (opener !== null) { + if (budgetSpent(ctx)) return exhaustedLiteral(ctx, input, valueStart); + const bodyStart = valueStart + opener[0].length; + const window = input.slice(bodyStart, bodyStart + MULTILINE_MAX_CHARS); + const found = new RegExp(String.raw`^${opener[1]}@`, 'm').exec(window); + if (found === null || !spendMultiline(ctx, found.index)) return exhaustedLiteral(ctx, input, bodyStart); + return { values: bodyValues(window.slice(0, found.index)), end: bodyStart + found.index + 1, exhausted: false }; + } + } + // A template / raw literal over several lines: JS `...`, Go `...`. (One line is read as an ordinary quoted value.) + if (first === '`' && ctx.mode === 'code') { + if (budgetSpent(ctx)) return exhaustedLiteral(ctx, input, valueStart); + const close = closingQuote(input, valueStart + 1, '`', { escapes: true }); + if (close === -1) return exhaustedLiteral(ctx, input, valueStart); + const body = input.slice(valueStart + 1, close); + if (!body.includes('\n')) return null; + if (!spendMultiline(ctx, body.length)) return exhaustedLiteral(ctx, input, valueStart); + return { values: bodyValues(body.replace(/\\\r?\n[ \t]*/g, '')).filter((line) => !line.includes('${')), end: close, exhausted: false }; + } + // A quote that is not closed on its line: shell NAME='ab', dotenv and YAML flow scalars over several lines. + if ((first === '"' || first === "'") && ctx.mode === 'config') { + const lineEnd = ctx.lineEnd(valueStart); + const sameLine = closingQuote(input, valueStart + 1, first, { escapes: first === '"', pair: first === "'", stop: lineEnd }); + if (sameLine !== -1) return null; + if (budgetSpent(ctx)) return exhaustedLiteral(ctx, input, valueStart); + const close = closingQuote(input, valueStart + 1, first, { escapes: first === '"', pair: first === "'" }); + if (close === -1 || !spendMultiline(ctx, close - valueStart)) return exhaustedLiteral(ctx, input, valueStart); + const body = input.slice(valueStart + 1, close); + return { values: bodyValues(first === '"' ? unescapeMultiline(body) : body.replace(/''/g, "'")), end: close, exhausted: false }; + } + return null; +} + +const ENDS_IN_BACKSLASH = /(?:^|[^\\])(?:\\\\)*\\$/; + +/** + * The lines that continue an unquoted value of a configuration file after its first line: + * - a trailing backslash (properties files, shell scripts): the next line follows without a break; + * - an INI value (configparser) or a YAML plain scalar folded over indented lines: the following lines indented deeper than the key. + * @returns {{values: string[], end: number, exhausted: boolean} | null} + */ +function continuedValue(ctx, input, matchIndex, valueStart) { + const lineEnd = ctx.lineEnd(valueStart); + const first = input.slice(valueStart, Math.min(lineEnd, valueStart + 4096)).replace(/\r$/, ''); + const pieces = []; + let end = lineEnd; + if (ENDS_IN_BACKSLASH.test(first)) { + if (budgetSpent(ctx)) return exhaustedLiteral(ctx, input, valueStart); + pieces.push(first.slice(0, -1)); + let at = lineEnd + 1; + let closed = false; + for (let n = 0; n < MULTILINE_MAX_LINES && at <= input.length; n += 1) { + let next = input.indexOf('\n', at); + if (next === -1) next = input.length; + const line = input.slice(at, Math.min(next, at + 4096)).replace(/\r$/, ''); + end = next; + const more = ENDS_IN_BACKSLASH.test(line); + pieces.push((more ? line.slice(0, -1) : line).trimStart()); + if (!more || next >= input.length) { + closed = true; + break; + } + at = next + 1; + } + if (!closed) return exhaustedLiteral(ctx, input, valueStart); + if (!spendMultiline(ctx, end - valueStart)) return exhaustedLiteral(ctx, input, valueStart); + return { values: [pieces.join(''), ...pieces.map((piece) => piece.trim()).filter((piece) => piece !== '')], end, exhausted: false }; + } + if (!/\.(?:ini|cfg|conf|ya?ml)$/i.test(ctx.path)) return null; + if (budgetSpent(ctx)) return exhaustedLiteral(ctx, input, valueStart); + const yaml = /\.ya?ml$/i.test(ctx.path); + const lineStart = ctx.lineStart(matchIndex); + const keyIndent = /^[ \t]*/.exec(input.slice(lineStart, matchIndex + 1))[0].length; + let at = lineEnd + 1; + for (let n = 0; n < 20 && at <= input.length; n += 1) { + let next = input.indexOf('\n', at); + if (next === -1) next = input.length; + const line = input.slice(at, Math.min(next, at + 4096)).replace(/\r$/, ''); + if (line.trim() === '' || /^[ \t]*[#;]/.test(line) || /^[ \t]*/.exec(line)[0].length <= keyIndent) break; + if (yaml && /^[ \t]*(?:[^\s#][^:]*:(?:[ \t]|$)|-[ \t])/.test(line)) break; // a mapping entry or a list item, not a folded scalar + pieces.push(line.trim()); + end = next; + if (next >= input.length) break; + at = next + 1; + } + if (pieces.length === 0) return null; + if (!spendMultiline(ctx, end - valueStart)) return exhaustedLiteral(ctx, input, valueStart); + return { values: [`${first.trim()} ${pieces.join(' ')}`.trim(), ...pieces], end, exhausted: false }; +} + +const hasFormat = (tag) => (ctx) => ctx.formats.has(tag); + +/** Any secret-like environment variable name, strong or weak. */ +const isSecretLikeName = (name) => secretNameKind(name) !== null; + +// --------------------------------------------------------------------------- +// The `name value` shape of secret-assignment +// --------------------------------------------------------------------------- + +// What may sit between a name and its assignment operator, by language: +// a type annotation: `: string`, `: &str`, `: &'static str`, `: []const u8`, `: String?` (TypeScript, Rust, Zig, Kotlin, Swift, Scala) +// a type word alone: `NAME string = ...` (Go var / const), `NAME char = ...` +// a marker: Nim `NAME* =`, TypeScript `name?: T =` / `name!: T =`, C `NAME[] =`, Lua `NAME =` +// Every part is bounded, starts with a character the previous part cannot end in, and is optional as a whole. +const ASSIGN_TYPE_COLON = String.raw`:[ \t]*(?:&(?:'[A-Za-z_]{1,16}[ \t]{1,4}|mut[ \t]{1,4})?|\*(?:const[ \t]{1,4}|mut[ \t]{1,4})?|\[\d{0,4}\](?:const[ \t]{1,4})?)?[A-Za-z_][A-Za-z0-9_<>[\]|.?!&*]{0,40}(?!:\/\/)`; +const ASSIGN_TYPE_WORD = String.raw`[ \t]{1,4}(?:\*|\[\d{0,4}\])?(?:string|String|str|byte|bytes|char|text|any|auto|dynamic|object|Object)(?![A-Za-z0-9_])`; +const ASSIGN_MARKER = String.raw`(?:\*|[?!](?=[ \t]*:)|\[[A-Za-z0-9_]{0,20}\]|[ \t]*<(?:const|close)>)`; +// Every assignment operator that carries a value: = := ::= ?= += -= .= *= **= |= &= ^= %= ||= &&= ??= => and the arrows <- <<- -> (R, Scala) and the +// infix `to` (Kotlin `"NAME" to "value"`). An arrow or `to` counts only in front of a quote, so `a->b`, `x<-1` and prose stay out. +const ASSIGN_OPERATOR = String.raw`(\|\|=|&&=|\?\?=|\*\*=|:{1,3}=|\?=|[-+.*|&^%]=|=>|<{1,2}-(?=[ \t]*["'\x60])|->(?=[ \t]*["'\x60])|(?<=["'\x60][ \t]{1,8})to(?=[ \t]{1,8}["'\x60])|=|:(?!:))`; +const ASSIGNMENT_PATTERN = new RegExp( + String.raw`(?(?:[^'\n]|''){1,4096})'|[Ee]'(?(?:[^'\\\n]|''|\\.){1,4096})'|"(?(?:[^"\n]|""){1,4096})"|\x60(?(?:[^\x60\n]|\x60\x60){1,4096})\x60|\$(?[A-Za-z_][A-Za-z0-9_]{0,32})?\$(?

(?:(?!\$\k\$)[^\n]){1,4096}?)\$\k\$)`; +// Oracle also takes the password as a bare identifier: IDENTIFIED BY hunter2 (only read inside a CREATE / ALTER / GRANT statement). +const SQL_BARE = String.raw`(?[A-Za-z][A-Za-z0-9_#$]{0,127})`; +// Words that follow IDENTIFIED BY and are syntax, not a password. +const SQL_SYNTAX_WORDS = new Set(['values', 'password', 'random', 'externally', 'globally', 'replace', 'default', 'null', 'using', 'with']); +// Bind parameters and substitution variables: ?, $1, :name, :1, @name, %s, %(name)s, %L, {0}, {name}, &pw (SQL*Plus). +const SQL_PARAMETER = + /^(?:\?|\$\d{1,3}|:[A-Za-z_][A-Za-z0-9_]{0,63}|:\d{1,3}|@[A-Za-z_][A-Za-z0-9_]{0,63}|%(?:\([A-Za-z_][A-Za-z0-9_]{0,63}\))?[sdLI]|\{\d{0,3}\}|\{[A-Za-z_][A-Za-z0-9_.]{0,63}\}|&&?[A-Za-z_][A-Za-z0-9_]{0,63}\.?)$/; + +/** The password literal of a SQL match, unquoted (doubled quotes undone), or undefined. */ +function sqlLiteral(groups) { + if (groups.sq !== undefined) return groups.sq.replace(/''/g, "'"); + if (groups.esc !== undefined) return groups.esc.replace(/''/g, "'").replace(/\\(.)/g, '$1'); + if (groups.dq !== undefined) return groups.dq.replace(/""/g, '"'); + if (groups.bt !== undefined) return groups.bt.replace(/\x60\x60/g, '\x60'); + return groups.dl ?? groups.id; +} + +const isSqlPath = (ctx) => /\.(?:sql|psql|pgsql|mysql|ddl)$/i.test(ctx.path); + +/** The statement text before the match (back to the previous `;`, at most 400 characters). */ +function sqlStatementBefore(m) { + const before = m.input.slice(Math.max(0, m.index - 400), m.index); + return before.slice(before.lastIndexOf(';') + 1); +} + +const sqlValueIsSecret = (value) => value !== undefined && !placeholderOf(value) && !SQL_PARAMETER.test(value.trim()); + +const SQL_RULE = { + id: 'sql-password-literal', + description: + "SQL statement that sets a role or user password to a literal (ALTER ROLE ... PASSWORD '...', CREATE USER ... IDENTIFIED BY \"...\", MongoDB createUser({pwd: \"...\"}))", + matchers: [ + { + // Oracle / MySQL / MariaDB: IDENTIFIED BY '' / "" / , IDENTIFIED WITH plugin BY '' | AS '', IDENTIFIED BY PASSWORD '', + // IDENTIFIED VIA plugin USING PASSWORD(''); also in GRANT ... IDENTIFIED BY. A hash is judged like any other literal. + hint: /identified/i, + pattern: new RegExp( + String.raw`\bidentified[ \t]+(?:(?:with|via)[ \t]+[A-Za-z_][A-Za-z0-9_]{0,63}[ \t]+(?:by|as|using)|by)[ \t]+(?:(?:values|password)[ \t]*(?:\([ \t]*)?)?(?:${SQL_QUOTED}|${SQL_BARE})`, + 'gi', + ), + accept: (m, ctx) => { + if (m.groups.id !== undefined) { + // A bare word is a password only in a statement (a .sql file, or CREATE / ALTER / GRANT before it), never in prose. + const id = m.groups.id.toLowerCase(); + if (SQL_SYNTAX_WORDS.has(id) || !(isSqlPath(ctx) || /\b(?:create|alter|grant)\b/i.test(sqlStatementBefore(m)))) return false; + } + return sqlValueIsSecret(sqlLiteral(m.groups)); + }, + }, + { + // PostgreSQL / SQL Server / Snowflake / MySQL: [CREATE|ALTER USER|ROLE|LOGIN ...] [WITH] [ENCRYPTED] PASSWORD [=] '', + // SET PASSWORD [FOR user] = '' | PASSWORD(''), OLD_PASSWORD = ''. A bare `password '...'` is only SQL in a .sql file. + hint: /password/i, + pattern: new RegExp( + String.raw`\b(?
(?:alter|create)[ \t]+(?:user|role|login|group|server)\b[^;'\n]{0,160}?\b|with[ \t]+(?:(?:encrypted|unencrypted)[ \t]+)?|login[ \t]+(?:(?:encrypted|unencrypted)[ \t]+)?|(?:encrypted|unencrypted)[ \t]+|old_|set[ \t]+)?password(?![A-Za-z0-9_])(?:[ \t]+(?for[ \t]+[^=;\s][^=;\n]{0,119})=[ \t]*|[ \t]*(?=[ \t]*)?(?<=[ \t=]))(?:(?:old_)?password[ \t]*\([ \t]*)?${SQL_QUOTED}`,
+        'gi',
+      ),
+      accept: (m, ctx) => {
+        const pre = m.groups.pre ?? '';
+        const qualified = /^(?:alter|create|with|login)\b/i.test(pre) || (/^set\b/i.test(pre) && (m.groups.for !== undefined || m.groups.eq !== undefined));
+        if (!qualified && !isSqlPath(ctx)) return false;
+        return sqlValueIsSecret(sqlLiteral(m.groups));
+      },
+    },
+    {
+      // MySQL / MariaDB: UPDATE mysql.user SET authentication_string = PASSWORD(''), ... USING PASSWORD('').
+      hint: /password[ \t]*\(/i,
+      pattern: new RegExp(String.raw`(? {
+        const before = m.input.slice(Math.max(0, m.index - 16), m.index);
+        if (!isSqlPath(ctx) && !/(?:=|\b(?:using|by|as))[ \t]*$/i.test(before)) return false;
+        return sqlValueIsSecret(sqlLiteral(m.groups));
+      },
+    },
+    {
+      // Oracle / MySQL: IDENTIFIED BY '' REPLACE '' (the old password is a password too).
+      hint: /replace/i,
+      pattern: new RegExp(String.raw`\breplace[ \t]+(?:${SQL_QUOTED}|${SQL_BARE})`, 'gi'),
+      accept: (m) => {
+        if (!/\bidentified\b/i.test(sqlStatementBefore(m))) return false;
+        if (m.groups.id !== undefined && SQL_SYNTAX_WORDS.has(m.groups.id.toLowerCase())) return false;
+        return sqlValueIsSecret(sqlLiteral(m.groups));
+      },
+    },
+    {
+      // MongoDB: db.createUser({user: "a", pwd: "", roles: [...]}), db.updateUser("a", {pwd: ""}); the key is `pwd`.
+      hint: /pwd/i,
+      pattern: new RegExp(String.raw`(? {
+        const near = ['createUser', 'updateUser', 'changeUserPassword', 'addUser'].some((call) => ctx.hasBefore(call, m.index, 2000));
+        return near && sqlValueIsSecret(sqlLiteral(m.groups));
+      },
+    },
+    {
+      // MongoDB legacy helpers: db.changeUserPassword("user", ""), db.addUser("user", ""), db.auth("user", "").
+      hint: /changeUserPassword|addUser|\.auth\(/,
+      pattern: new RegExp(String.raw`\b(?:changeUserPassword|addUser|db\.auth)[ \t]*\([ \t]*(?:"[^"\n]{1,200}"|'[^'\n]{1,200}')[ \t]*,[ \t]*${SQL_QUOTED}`, 'g'),
+      accept: (m) => sqlValueIsSecret(sqlLiteral(m.groups)),
+    },
+  ],
+};
+
+/**
+ * Each rule: id, description, matchers[{ pattern (global regex), accept(match, ctx), optional appliesTo(ctx) and hint }],
+ * optional appliesTo(ctx) and hint (a cheap regex the file must match before the rule runs).
+ * ctx is { path, mode, formats, strict, minStrong }. accept() returns false for placeholders and other non-secrets.
+ * Every quantifier that can meet attacker-shaped text is bounded, so scan time stays linear.
+ */
+
+// Provider-specific token shapes. Each is recognised by its fixed prefix and length wherever it appears (any file, any
+// name), so a token under a non-secret name or in prose is found. Boundaries use lookbehind and a class that cannot
+// overlap the prefix, so each match attempt is bounded and the scan stays linear. `lockfile: true`: these also run on
+// lockfiles. A value that is a documentation placeholder (xxxx, your_..., <...>) passes.
+// Long token bodies are judged by their head and tail (a placeholder marker is at the ends), so the cost of the placeholder
+// check does not grow with an attacker-sized match.
+const placeholderOf = (text) => isPlaceholder(text.length > 512 ? text.slice(0, 256) + text.slice(-256) : text);
+const notPlaceholder = (m) => !placeholderOf(m[0]);
+const mixedCase = (text) => /[A-Z]/.test(text) && /[a-z]/.test(text);
+// A run of one or two repeated characters (000000...) is a placeholder, not a hex token.
+const varied = (text) => new Set(text).size >= 8;
+const hasDigitAndLetter = (text) => /\d/.test(text) && /[A-Za-z]/.test(text);
+const tokenRule = (id, description, hint, patterns, accept = notPlaceholder) => ({
+  id,
+  description,
+  lockfile: true,
+  hint,
+  matchers: (Array.isArray(patterns) ? patterns : [patterns]).map((pattern) => ({ pattern, accept })),
+});
+const PROVIDER_RULES = [
+  tokenRule(
+    'gitlab-token',
+    'GitLab personal, deploy, runner, trigger, feed or agent token (glpat-, gldt-, glrt-, ...)',
+    /gl[a-z]{2,6}-/,
+    /(? !placeholderOf(m[0]) && hasDigitAndLetter(m[0].slice(4))),
+  tokenRule('pypi-token', 'PyPI API token (pypi- prefix)', /pypi-/, /(? !placeholderOf(m[0]) && varied(m[0])),
+  tokenRule(
+    'sendgrid-api-key',
+    'SendGrid API key (SG. prefix)',
+    /SG\./,
+    /(? !placeholderOf(m[0]) && varied(m[0].slice(3)),
+  ),
+  tokenRule('mailgun-api-key', 'Mailgun private API key (key- + 32 hex)', /key-[0-9a-f]{32}/, /(? !placeholderOf(m[0]) && varied(m[0])),
+  tokenRule(
+    'shopify-token',
+    'Shopify access token (shpat_, shpca_, shppa_, shpss_)',
+    /shp(?:at|ca|pa|ss)_/,
+    /(? !placeholderOf(m[0]) && varied(m[0]),
+  ),
+  tokenRule(
+    'digitalocean-token',
+    'DigitalOcean token (dop_v1_, doo_v1_, dor_v1_)',
+    /do[opr]_v1_/,
+    /(? !placeholderOf(m[0]) && varied(m[0]),
+  ),
+  tokenRule(
+    'huggingface-token',
+    'Hugging Face access token (hf_ prefix)',
+    /hf_|api_org_/,
+    /(? !placeholderOf(m[0]) && mixedCase(m[0]),
+  ),
+  tokenRule(
+    'openai-api-key',
+    'OpenAI API key (project, service-account, admin and legacy keys)',
+    /sk-/,
+    [
+      /(? !placeholderOf(m[0]) && (m[0].includes('-proj-') || m[0].includes('-svcacct-') || m[0].includes('-admin-') || (mixedCase(m[0]) && /\d/.test(m[0]))),
+  ),
+  tokenRule('anthropic-api-key', 'Anthropic API key (sk-ant- prefix)', /sk-ant-/, /(? !placeholderOf(m[1]) && /\d/.test(m[1]) && mixedCase(m[1]),
+      },
+      {
+        // A SAS URL: ...?sv=2022-11-02&ss=b&srt=sco&sp=rwl&se=2030-01-01T00:00:00Z&sig=
+        pattern: /[?&;]sig=([A-Za-z0-9%+/_-]{40,})/g,
+        accept: (m) => {
+          const around = m.input.slice(Math.max(0, m.index - 400), m.index + 400);
+          return /[?&;](?:sv|se|sp|srt|ss|spr)=/.test(around) && !placeholderOf(m[1]);
+        },
+      },
+    ],
+  },
+  {
+    id: 'heroku-api-key',
+    description: 'Heroku authorization token (HRKU- prefix) or HEROKU_API_KEY set to a UUID',
+    lockfile: true,
+    hint: /HRKU-|HEROKU/i,
+    matchers: [
+      { pattern: /(? !placeholderOf(m[1]) && new Set(m[1]).size > 6,
+      },
+    ],
+  },
+  {
+    id: 'datadog-api-key',
+    description: 'Datadog API or application key (32 or 40 hex) set on a Datadog key name',
+    lockfile: true,
+    hint: /(?:DD|DATADOG)[_-](?:API|APP)/i,
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[1]) && new Set(m[1].toLowerCase()).size > 6,
+      },
+    ],
+  },
+  {
+    id: 'sentry-token',
+    description: 'Sentry auth token (sntrys_, sntryu_) or a DSN carrying its secret half (key:secret@)',
+    lockfile: true,
+    hint: /sntry|sentry/i,
+    matchers: [
+      { pattern: /(?:<32 hex secret>@o123.ingest.sentry.io/456. Only a DSN that still carries the deprecated SECRET
+        // half is a credential: the public key of a modern DSN ships in every browser bundle and is safe to expose by design.
+        pattern: /(? !placeholderOf(m[1]) && new Set(m[1]).size > 6,
+      },
+    ],
+  },
+  tokenRule(
+    'doppler-token',
+    'Doppler token (dp.st., dp.pt., dp.ct., dp.sa., ...)',
+    /dp\./,
+    /(? !placeholderOf(m[0]) && hasDigitAndLetter(m[0].slice(4)),
+  ),
+  tokenRule('linear-api-key', 'Linear API key or OAuth token', /lin_/, /(? !placeholderOf(m[0]) && (m[0].startsWith('ntn_') || (hasDigitAndLetter(m[0].slice(7)) && mixedCase(m[0].slice(7)))),
+  ),
+  tokenRule(
+    'atlassian-token',
+    'Atlassian API token (ATATT) or Bitbucket app password (ATBB)',
+    /ATATT|ATBB/,
+    /(?:AA + 33 characters)',
+    /:AA/,
+    /(? !placeholderOf(m[0]) && varied(m[0].slice(m[0].indexOf(':') + 1)),
+  ),
+  tokenRule(
+    'mapbox-secret-token',
+    'Mapbox secret access token (sk.eyJ...)',
+    /sk\.eyJ/,
+    /(? !placeholderOf(m[0]) && varied(m[0].slice(7)),
+  ),
+  tokenRule(
+    'firebase-fcm-server-key',
+    'Firebase Cloud Messaging legacy server key (AAAA...:APA91b...)',
+    /APA91b/,
+    /(? !placeholderOf(m[0]) && varied(m[0].slice(5)),
+  ),
+  tokenRule(
+    'cloudflare-token',
+    'Cloudflare API token (cfut_, cfat_, cfk_ prefix)',
+    /cf(?:ut|at|k)_/,
+    /(? !placeholderOf(m[0]) && varied(m[0].slice(5)),
+  ),
+  tokenRule(
+    'discord-bot-token',
+    'Discord bot token (..)',
+    /\.[A-Za-z0-9_-]{6}\./,
+    /(? !placeholderOf(m[0]) && mixedCase(m[0]) && /\d/.test(m[0]) && varied(m[0]),
+  ),
+  tokenRule(
+    'other-provider-token',
+    'Other provider token (Databricks, Grafana, Supabase, PlanetScale, Docker Hub, RubyGems, Terraform Cloud, age secret key)',
+    /dapi|glsa_|sbp_|pscale_|dckr_pat_|rubygems_|atlasv1|AGE-SECRET-KEY/,
+    [
+      /(? !placeholderOf(m[0]) && varied(m[0]),
+  ),
+];
+
+// Lockfile fields that can hold a credential. The generic secret-name rules do not run on lockfiles (integrity hashes), so
+// this rule looks only at URLs (userinfo token, credential-named query parameter) and at auth-like FIELD names.
+const LOCKFILE_URL = /(?\\]{1,2000}/gi;
+const LOCKFILE_AUTH_FIELD =
+  /(?=1 <2", "1.x", "*",
+// "npm:x@1", "workspace:*", "link:../x", "latest"): a first character that merely looks like one (a digit, x, v) exempts nothing.
+const SPEC_VERSION = String.raw`[\^~<>=]{0,2}[ \t]?v?\d{1,8}(?:\.[\dxX*]{1,12}){0,3}(?:-(?:alpha|beta|rc|dev|next|canary|pre|preview|nightly|snapshot|experimental)(?:[.-]?\d{1,4}){0,4})?`;
+const LOCKFILE_SPEC_VALUE = new RegExp(
+  String.raw`^(?:${SPEC_VERSION}(?:(?:[ \t]*(?:\|\||-|,)[ \t]*|[ \t]+)${SPEC_VERSION}){0,4}|[\^~<>=]{0,2}[ \t]?[xX*]|(?:\.{1,2}|~)?(?:\/[A-Za-z0-9._@-]{1,64}){1,12}\/?|\.{1,2}|(?:npm|file|link|workspace|git|github|gitlab|bitbucket|patch|portal|catalog|resolution):[^\s]{0,256}|(?:latest|next|beta|alpha|canary|rc|true|false|null|none|undefined))$`,
+);
+// A bare "key" is not listed: cache-key, sort-key and the like are identifiers; only credential-qualified keys are.
+const LOCKFILE_CREDENTIAL_PARAM = /(?:sig|signature|token|secret|(?:api|access|secret|private|auth|signing)[-_.]?key|password|passwd|pwd|auth|authorization|jwt|bearer|sas)$/i;
+
+/** Does a URL in a lockfile carry a credential of its own: a token as the user name, or a credential-named query parameter? */
+function lockfileUrlCarriesCredential(url) {
+  const rest = url.replace(URL_PREFIX, '');
+  const authorityEnd = rest.search(/[/?#]/);
+  const authority = authorityEnd === -1 ? rest : rest.slice(0, authorityEnd);
+  const at = authority.lastIndexOf('@');
+  if (at > 0) {
+    const user = authority.slice(0, authority.indexOf(':') === -1 || authority.indexOf(':') > at ? at : authority.indexOf(':'));
+    // ssh://git@host, https://user@host and token-style names are ordinary; a long random user name is a token.
+    if (user.length >= 16 && !isTemplateSegment(user) && !isWordIdentifier(user) && looksRandom(user, { minLength: 16, minEntropy: 3.2 })) return true;
+  }
+  if (authorityEnd === -1) return false;
+  const afterAuthority = rest.slice(authorityEnd);
+  const queryAt = afterAuthority.indexOf('?');
+  if (queryAt === -1) return false;
+  const hashAt = afterAuthority.indexOf('#', queryAt);
+  const query = afterAuthority.slice(queryAt + 1, hashAt === -1 ? undefined : hashAt);
+  for (const pair of query.split(/[&;]/)) {
+    const eq = pair.indexOf('=');
+    if (eq <= 0) continue;
+    const name = safeDecode(pair.slice(0, eq));
+    const value = safeDecode(pair.slice(eq + 1));
+    if (LOCKFILE_CREDENTIAL_PARAM.test(name) && value.length >= 8 && !isTemplateSegment(value) && !isWordIdentifier(value)) return true;
+  }
+  return false;
+}
+
+const LOCKFILE_RULE = {
+  id: 'lockfile-credential',
+  description:
+    'credential inside a lockfile: a token or password in a dependency URL (resolved, tarball, url, source, registry) or an auth field such as _authToken, _auth, _password',
+  lockfile: true,
+  appliesTo: (ctx) => ctx.lockfile,
+  matchers: [
+    // A YAML flow mapping or a JSON array can end right after the URL: {tarball: https://h/x?sig=V} is the URL without the brace.
+    { pattern: LOCKFILE_URL, accept: (m) => lockfileUrlCarriesCredential(m[0].replace(/[)}\],;]+$/, '')) },
+    {
+      pattern: LOCKFILE_AUTH_FIELD,
+      accept: (m, ctx) => {
+        // The name must be a whole key: a word before it on the line makes it prose ("CSRF token: generation and ...").
+        const before = m.input.slice(ctx.lineStart(m.index), m.index).trimEnd();
+        if (/[A-Za-z0-9]$/.test(before)) return false;
+        // Free-text package metadata (composer.lock, poetry.lock) is not an auth field.
+        if (/^[ \t]*["']?(?:description|summary|homepage|title|readme|keywords|notes?)["']?[ \t]*[:=]/i.test(before)) return false;
+        const value = (m[2] ?? m[3] ?? m[4]).trim();
+        if (value.length < 4 || LOCKFILE_SPEC_VALUE.test(value) || URL_PREFIX.test(value)) return false; // a URL value is judged by the URL matcher
+        return !isPlaceholder(value);
+      },
+    },
+  ],
+};
+
+// The attribute that names an entry: V, V.
+const XML_NAME_ATTRIBUTE = /(?:^|[ \t\r\n])(?:name|key)[ \t]*=[ \t]*(?:"([^"]{1,200})"|'([^']{1,200})')/i;
+// A file-system path (/path/to/private/key, ./certs/key.pem, ~/.ssh/id_rsa): a key FILE element holds this, not a key.
+const PATH_VALUE = /^(?:\.{1,2}|~)?(?:\/[A-Za-z0-9._@-]{1,64}){2,12}\/?$/;
+// Example values Maven's reference settings.xml documents; they are sample text, not credentials.
+// (The passphrase hint is exact text: a sentence is not exempt by its punctuation, only this whole value is.)
+const XML_EXAMPLE_VALUES = new Set(['proxypass', 'optional; leave empty if not used.']);
+
+/** Is the text of this XML element (name, start-tag attributes, text) a secret? See the element matchers in RULES. */
+function xmlElementIsSecret(qualifiedName, attributes, text, ctx) {
+  const name = qualifiedName.slice(qualifiedName.lastIndexOf(':') + 1);
+  let kind = nameKindFor(name, ctx);
+  if (kind === null) {
+    const attribute = XML_NAME_ATTRIBUTE.exec(attributes);
+    if (attribute !== null) kind = nameKindFor(attribute[1] ?? attribute[2], ctx);
+  }
+  if (kind === null) return false;
+  const value = text.trim();
+  if (value === '') return false;
+  // A Maven-encrypted password ({base64}) is not a plaintext credential: it is unreadable without the master key.
+  if (/^\{[A-Za-z0-9+/=]{20,}\}$/.test(value)) return false;
+  if (PATH_VALUE.test(value) || XML_EXAMPLE_VALUES.has(value.toLowerCase())) return false;
+  // Text about the credential ("the password you chose during setup.") is exempt through the same documentation-cue rule
+  // as every other format; punctuation alone never is, or a passphrase ending in "!" would slip through.
+  return isSecretValue({ kind, value, quoted: true, separator: ':', mode: ctx.mode, minLength: ctx.minStrong, catalog: ctx.catalog });
+}
+
+/** A multi-line value that is an ansible-vault ciphertext ($ANSIBLE_VAULT;1.1;AES256 and its hex lines): encrypted, so not a credential. */
+const isVaultBody = (values) => values.length > 0 && /^\$ANSIBLE_VAULT[;\s]/.test(values[0]);
+
+const SECRET_ASSIGNMENT_MATCHER = {
+  // group 2 = name, 3 = separator. The VALUE is deliberately not part of the match: it is read in accept()
+  // (VALUE_AT) and only for a secret-like name. A rejected match therefore consumes nothing but `name =`, and
+  // matching resumes right after it, so `cfg['a']={"K":"v"}`, `x=1;K='v'` and minified JSON are still examined.
+  // Not a name: "${NAME:-x}" (an expansion, judged by its outer assignment) or "://NAME:x@" (a URL, judged by url-password).
+  pattern: ASSIGNMENT_PATTERN,
+  accept: (m, ctx) => {
+    const kind = nameKindFor(m[2], ctx);
+    if (!kind) return false;
+    const input = m.input;
+    let valueStart = m.index + m[0].length;
+    if (ctx.mode !== 'code' && m[3] === ':') {
+      // A YAML tag or anchor before the scalar (`!!str V`, `&anchor V`, `!vault V`) is not part of the value.
+      YAML_NODE_PROPERTIES.lastIndex = valueStart;
+      const properties = YAML_NODE_PROPERTIES.exec(input);
+      if (properties) {
+        // `!vault |` marks an ansible-vault ciphertext: encrypted, not a plaintext credential.
+        if (/(?:^|[ \t])!vault[ \t]/.test(properties[0])) return false;
+        valueStart += properties[0].length;
+      }
+    }
+    // A literal over several lines (heredoc, triple quotes, a quote closed on a later line, a template literal) is judged
+    // as a whole. One that does not end within the bounds is reported: it cannot be verified.
+    const multiline = multilineValue(ctx, input, valueStart);
+    if (multiline !== null) {
+      m.spanEnd = multiline.end;
+      if (multiline.exhausted) return exhaustedVerdict(m, ctx, multiline);
+      if (isVaultBody(multiline.values)) return false;
+      return multiline.values.some((text) =>
+        isSecretValue({ kind, value: text, quoted: true, separator: m[3], mode: ctx.mode, minLength: ctx.minStrong, catalog: ctx.catalog }),
+      );
+    }
+    // Assignments nested in one whitespace-free run (`a=b=c=...`) all share its tail. The first value gets the
+    // full length; nested ones are judged on their first 64 characters, which keeps a hostile run linear.
+    const nested = valueStart < (ctx.bareRunEnd ?? 0);
+    const reader = nested ? NESTED_VALUE_AT : VALUE_AT;
+    reader.lastIndex = valueStart;
+    const value = reader.exec(input);
+    if (!value) return false;
+    const quotedValue = value[1] ?? value[2] ?? value[3];
+    const quoted = quotedValue !== undefined;
+    m.spanEnd = valueStart + value[0].length;
+    // `NAME=\`cat file\`` in a shell script is a command substitution, not a string literal (in code it is a template literal).
+    if (value[3] !== undefined && ctx.mode !== 'code' && isShellScriptPath(ctx.path)) return false;
+    const check = (text, isQuoted) =>
+      isSecretValue({ kind, value: text, quoted: isQuoted, separator: m[3], mode: ctx.mode, minLength: ctx.minStrong, catalog: ctx.catalog });
+    if (quoted) return check(unescapeQuoted(quotedValue), true);
+    const bare = value[4];
+    if (!nested) ctx.bareRunEnd = valueStart + bare.length;
+    let token = bare.replace(/^["'`]+/, '');
+    if (ctx.mode === 'config' && bare.includes('${')) {
+      // The value is the whole shell word, so ${A:-two words} and ${A}suffix stay in one piece.
+      const wordEnd = bareShellWordEnd(input, valueStart);
+      token = input.slice(valueStart, wordEnd);
+      m.spanEnd = Math.max(m.spanEnd, wordEnd);
+    }
+    if (bare.startsWith('`') && ctx.mode !== 'code' && isShellScriptPath(ctx.path)) return false; // an unquoted command substitution
+    if (ctx.mode !== 'config') return check(token, false);
+    const extras = unquotedContinuations(ctx, input, m.index, valueStart + bare.length, bare);
+    if (extras.some((extra) => isVaultBody([extra.value]))) return false; // `NAME: |` + an ansible-vault ciphertext
+    // A backslash at the end of the line, an INI continuation line, a YAML scalar folded over indented lines.
+    const continued = /^[|>][-+0-9]*$/.test(bare) ? null : continuedValue(ctx, input, m.index, valueStart);
+    if (continued !== null) {
+      m.spanEnd = continued.end;
+      if (continued.exhausted) return exhaustedVerdict(m, ctx, continued);
+      if (isVaultBody(continued.values)) return false;
+      for (const text of continued.values) if (check(text, true)) return true;
+    }
+    // A value that continues after its first word (`password=Passwords do not match`) is the whole rest of the line:
+    // the first word alone says nothing about it.
+    const wholeLine = extras.find((extra) => extra.value.length > bare.length && extra.value.startsWith(bare));
+    if (wholeLine) {
+      m.spanEnd = wholeLine.end;
+      return check(wholeLine.value, true);
+    }
+    if (check(token, false)) return true;
+    for (const extra of extras) {
+      if (check(extra.value, true)) {
+        m.spanEnd = extra.end;
+        return true;
+      }
+    }
+    return false;
+  },
+};
+
+
+// Definition forms with no operator, where the name follows a keyword or sigil and the value is a quoted literal after a blank:
+// Elixir `@jwt_secret "v"`, Clojure `(def jwt-secret "v")` / `{:jwt-secret "v"}`, Lisp `(setq jwt-secret "v")` / `(defvar ...)`.
+// Source code only (in prose these read as ordinary text); the value is judged like any other quoted literal in code.
+const DEFINITION_ASSIGNMENT_MATCHER = {
+  appliesTo: (ctx) => ctx.mode === 'code',
+  hint: /[@(:]/,
+  pattern:
+    /(? SECRET_ASSIGNMENT_MATCHER.accept(m, ctx),
+};
+
+// Java properties: `key value` is a line too. The key ends at the first unescaped blank, `=` or `:`, and the value is the rest of the
+// line (with `\` continuations); `key=value` and `key:value` are read by the assignment matchers above. The key is a run of
+// characters that are not blanks, `=`, `:` or a backslash, or a backslash and the character it escapes, so each character has one reading.
+// A `#` or `!` first character is a comment.
+const PROPERTIES_BLANK_MATCHER = {
+  appliesTo: (ctx) => ctx.properties,
+  pattern: /^[ \t\f]{0,64}((?:[^\s=:#!\\]|\\[^\n]){1}(?:[^\s=:\\]|\\[^\n]){0,1023})[ \t\f]{1,64}(?=[^\s=:])/gm,
+  accept: (m, ctx) => {
+    const kind = nameKindFor(m[1].replace(/\\(.)/g, '$1'), ctx);
+    if (!kind) return false;
+    const input = m.input;
+    const valueStart = m.index + m[0].length;
+    const lineEnd = ctx.lineEnd(valueStart);
+    const first = input.slice(valueStart, Math.min(lineEnd, valueStart + 4096)).replace(/\r$/, '');
+    m.spanEnd = Math.min(lineEnd, valueStart + first.length);
+    const check = (text, quoted) => isSecretValue({ kind, value: text, quoted, separator: ' ', mode: ctx.mode, minLength: ctx.minStrong, catalog: ctx.catalog });
+    const continued = continuedValue(ctx, input, m.index, valueStart);
+    if (continued !== null) {
+      m.spanEnd = continued.end;
+      if (continued.exhausted) return exhaustedVerdict(m, ctx, continued);
+      return continued.values.some((text) => check(text, true));
+    }
+    const token = first.trimEnd();
+    if (/\s/.test(token)) return check(token, true) || check(token.split(/\s/, 1)[0], false);
+    return check(token, false);
+  },
+};
+
+// ---------------------------------------------------------------------------
+// HTTP credentials written out: `Authorization: Bearer `, headers.set('Authorization', 'Basic '),
+// `proxy_set_header Authorization "Bearer "`, requests.get(url, auth=('user', 'password')), ...
+// ---------------------------------------------------------------------------
+
+// A token in a header must be at least this long to count: shorter text is a scheme keyword, a label or a test value.
+const HTTP_CREDENTIAL_MIN_LENGTH = 12;
+const HTTP_TOKEN_GATE = { minLength: HTTP_CREDENTIAL_MIN_LENGTH, minEntropy: 3.0 };
+// The examples of RFC 7617, 6749, 6750 and 5849 (Aladdin / open sesame, the OAuth client, and the two sample bearer tokens): documentation, in any file.
+const HTTP_SAMPLE_LOGINS = new Set(['aladdin:open sesame', 's6bhdrkqt3:gx1fbat3bv']);
+const HTTP_SAMPLE_TOKENS = new Set(['mf_9.b5f-4.1jqm', 'h480djs93hd8']);
+// A value that stands for a token (${token}, {token}, $(cat f), , [token], f(x)) contains one of these; real tokens do not.
+const HTTP_TOKEN_REFERENCE = /[{}()<>[\]$]/;
+
+/** The password of a decoded `user:password` pair, or null when `value` is not the base64 of a printable pair. */
+function basicPassword(value) {
+  if (!/^[A-Za-z0-9+/_-]{4,4096}={0,2}$/.test(value)) return null;
+  const decoded = Buffer.from(value.replace(/-/g, '+').replace(/_/g, '/'), 'base64').toString('utf8');
+  if (!/^[\x20-\x7e]{3,4096}$/.test(decoded)) return null;
+  if (HTTP_SAMPLE_LOGINS.has(decoded.toLowerCase())) return '';
+  const colon = decoded.indexOf(':');
+  return colon === -1 ? null : decoded.slice(colon + 1);
+}
+
+/**
+ * Is the credential of an HTTP authorization header (`Bearer `, `Basic `, `token `, a bare token) a secret?
+ * A reference or placeholder is not. Basic credentials are decoded and the password is judged like a URL password
+ * (so `user:pass` and the RFC sample pass); everything else must look machine-generated. A JWT is the jwt-token rule's business.
+ */
+function httpCredentialIsSecret(scheme, raw) {
+  const value = raw.replace(/[,;.]+$/, '');
+  if (value.includes('${')) return expansionLiterals(value).some((literal) => looksRandom(stripQuotes(literal), HTTP_TOKEN_GATE));
+  if (HTTP_TOKEN_REFERENCE.test(value) || isPlaceholder(value)) return false;
+  if (/^eyJ[A-Za-z0-9_-]{10,}\.eyJ/.test(value) || HTTP_SAMPLE_TOKENS.has(value.toLowerCase())) return false;
+  if (scheme !== undefined && scheme.toLowerCase() === 'basic') {
+    const password = basicPassword(value);
+    if (password !== null) return password !== '' && urlPasswordIsSecret(password);
+  }
+  return looksRandom(value, HTTP_TOKEN_GATE);
+}
+
+// What follows the header name (all groups are shared by the matchers below):
+//   1 = opening quote of the value ('' when there is none), 2 = the scheme (Bearer, Basic, token, ApiKey, ...), 3 = the credential
+// The credential stops at a blank, a quote or a backslash, so `Bearer ` inside "..." or a JSON string is read whole.
+const HTTP_CREDENTIAL = String.raw`(["'\x60]?)(?:([A-Za-z][A-Za-z0-9_-]{0,30})[ \t]{1,8})?([^\s"'\x60\\]{${HTTP_CREDENTIAL_MIN_LENGTH},4096})`;
+// [Proxy-]Authorization / X-Authorization
+const HTTP_AUTH_NAME = String.raw`(? ...` | `.set('Authorization', ...)`
+const HTTP_AUTH_SEPARATOR = String.raw`(?:["'\x60]?\]?[ \t]{0,8}(?:=>|[:=])|["'\x60][ \t]{0,8},)[ \t]{0,8}`;
+// Directives of servers and proxies that set a request header: nginx, Apache mod_headers, HAProxy.
+const HTTP_HEADER_DIRECTIVE = String.raw`(?(?:[^"\\\n]|\\.){0,4096})"|'(?[^'\n]{0,4096})'|(?[^\s"']{1,4096}))`;
+// Documented sample values (redis.conf ships `# requirepass foobared`).
+const DIRECTIVE_SAMPLES = new Set(['foobared', 'changeit']);
+
+/** The text of a DIRECTIVE_VALUE match ({ text, quoted }). A trailing `;` ends an nginx directive; it is not part of the value. */
+function directiveValue(groups) {
+  if (groups.dq !== undefined) return { text: unescapeQuoted(groups.dq), quoted: true };
+  if (groups.sq !== undefined) return { text: groups.sq, quoted: true };
+  return { text: groups.bare.replace(/;+$/, ''), quoted: false };
+}
+
+// nginx variables ($http_x_api_key, ${var}), HAProxy sample fetches and log-format (%[req.hdr(x)]), Apache expressions (%{HTTP_X}e): references.
+const DIRECTIVE_REFERENCE = /^(?:\$[A-Za-z_{(]|%[[{])/;
+
+/** Is the literal value of a service directive a secret? Strong-name rules: any non-placeholder of 8+ characters. */
+function directiveIsSecret(value) {
+  if (DIRECTIVE_SAMPLES.has(value.text.toLowerCase()) || DIRECTIVE_REFERENCE.test(value.text)) return false;
+  return isSecretValue({ kind: 'strong', value: value.text, quoted: value.quoted, separator: '=', mode: 'config' });
+}
+
+/** Directives are read in configuration files, and in fenced blocks of Markdown; anywhere else the text is prose or code. */
+const directiveContextOk = (m, ctx) => ctx.mode === 'config' || (ctx.mode === 'prose' && ctx.fenceLang(m.index) !== undefined);
+
+// Services whose configuration file is recognised by name (lower-case base name, template suffix removed).
+const SERVICE_CONFIG_FILES = [
+  ['redis', /^(?:redis|sentinel|valkey|keydb)[a-z0-9._-]{0,60}\.conf$/],
+  ['mosquitto', /^mosquitto[a-z0-9._-]{0,60}\.conf$/],
+  ['msmtp', /^\.?(?:msmtprc|mpoprc)$/],
+  ['fetchmail', /^\.?fetchmailrc$/],
+  ['tinyproxy', /^tinyproxy[a-z0-9._-]{0,60}\.conf$/],
+];
+
+/** Which service configuration a file is (see SERVICE_CONFIG_FILES), or null; also by directory (`.../mosquitto/*.conf`). */
+function serviceConfigOf(filePath) {
+  const normalized = filePath.split(path.sep).join('/').toLowerCase();
+  const base = baseWithoutTemplateSuffix(path.posix.basename(normalized));
+  for (const [service, pattern] of SERVICE_CONFIG_FILES) if (pattern.test(base)) return service;
+  const dir = /(?:^|\/)(redis|mosquitto)\/[^/]{1,100}\.conf$/.exec(normalized);
+  return dir === null ? null : dir[1];
+}
+
+/** Redis ACL rules: `>plaintext` adds a password, `#` a password hash. Other rules (on, ~*, +@all, nopass) are not secrets. */
+function aclRulesHoldSecret(rules) {
+  const tokens = rules.trim().split(/[ \t]+/);
+  // `user NAME` + rules: without an on/off/key/command rule the line is something else (an nginx `user` directive, prose).
+  if (!tokens.some((token) => /^(?:on|off|reset|resetpass|nopass|allkeys|allcommands|allchannels|~.+|%[RW]{1,2}~.+|\+.+|-.+|&.+)$/i.test(token))) return false;
+  return tokens.some((token) => {
+    if (token.startsWith('>')) return directiveIsSecret({ text: token.slice(1), quoted: false });
+    return /^#[0-9a-fA-F]{64}$/.test(token) && !isPlaceholder(token.slice(1));
+  });
+}
+
+export const RULES = [
+  {
+    id: 'url-password',
+    lockfile: true,
+    description: 'URL (or curl -u) with an embedded non-placeholder password: database, broker, HTTP basic auth, ...',
+    matchers: [
+      {
+        // No length caps on user and password: a long one must not make the match fail. Each attempt starts at a
+        // "://" and cannot cross a "/", so the scan stays linear. Only the host (never judged) is capped.
+        pattern: /(?<=[a-z0-9+.-]):\/\/([^\s:@/'"`]*):([^\s@/'"`]+)@([^\s/'"`?#]{0,256})/gi,
+        accept: (m) => urlPasswordIsSecret(m[2]) && !isDocumentationHost(m[3]),
+      },
+      {
+        // curl -u user:password https://...   (also --user, --proxy-user, -U, a flag cluster such as -sSu, and the attached
+        // forms -uuser:password and --user=user:password). The argument is read as one shell word, so a quoted
+        // "user:pass phrase" is judged whole (double, single and $'...' quotes, escaped spaces).
+        pattern: /(? {
+          if (!/curl|wget|https?:\/\//i.test(nearbyLineText(m))) return false;
+          const argument = shellArgument(m.input, m.index + m[0].length);
+          if (argument === null) return false;
+          const colon = argument.text.indexOf(':');
+          if (colon < 1) return false;
+          m.spanEnd = m.index + m[0].length + argument.length;
+          const password = argument.text.slice(colon + 1);
+          return urlPasswordIsSecret(argument.quoted ? password : password.replace(/[,;)]+$/, ''));
+        },
+      },
+      {
+        // Other tools that take a password as a flag argument: wget, mysql, mongosh, redis-cli, httpie and xh, sshpass,
+        // ldapsearch, curl --pass. The whole argument is judged, quoted or not.
+        pattern: CLI_PASSWORD_COMMAND,
+        accept: (m) => cliPasswordArguments(shellWords(m[0])).some((password) => urlPasswordIsSecret(password.replace(/[,;)]+$/, ''))),
+      },
+    ],
+  },
+  {
+    id: 'private-key-block',
+    lockfile: true,
+    description: 'PEM private key block (header followed by key material)',
+    matchers: [
+      {
+        pattern: new RegExp(
+          String.raw`-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----` +
+            PEM_SEPARATOR +
+            String.raw`(?:(?:Proc-Type|DEK-Info|Version|Comment):[^\n]*` +
+            PEM_SEPARATOR +
+            String.raw`)*([A-Za-z0-9+/=]{20,}[^\n]*)`,
+          'g',
+        ),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+    ],
+  },
+  {
+    id: 'aws-access-key-id',
+    lockfile: true,
+    description: 'AWS access key ID',
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[0]),
+      },
+    ],
+  },
+  {
+    id: 'google-api-key',
+    lockfile: true,
+    description: 'Google API key (also matches Firebase web API keys)',
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[0]),
+      },
+    ],
+  },
+  {
+    id: 'github-token',
+    lockfile: true,
+    description: 'GitHub personal access, OAuth, app or fine-grained token',
+    matchers: [
+      {
+        pattern:
+          /(? !placeholderOf(m[0]),
+      },
+    ],
+  },
+  {
+    id: 'slack-token',
+    lockfile: true,
+    description: 'Slack API token (bot, user, legacy, app-level xapp-, configuration and refresh tokens)',
+    matchers: [
+      {
+        // Bot/user/legacy/workspace/client tokens (xoxb xoxp xoxa xoxr xoxs xoxc xoxd), the rotating-token wrappers
+        // (xoxe.xoxp-, xoxe.xoxb-) and app-level tokens (xapp-). A real token always has digits (team and app ids); the
+        // lookahead is bounded, and the class after it cannot overlap the prefix, so a hostile run of "xoxb-xoxb-..." stays
+        // linear. A repeated-character body (xoxb-000000000000-...-xxxxxxxx) is a placeholder.
+        pattern: /(? !placeholderOf(m[0]) && varied(m[0]),
+      },
+      {
+        // Documented shapes only, so prose slugs ("xoxo-love-and-kisses-2026") are not tokens: Enterprise Grid xoxo--
+        // --, and configuration / refresh tokens xoxe--.
+        pattern: /(? !placeholderOf(m[0]) && varied(m[0]) && (m[0].startsWith('xoxo-') || (mixedCase(m[0].slice(7)) && /\d/.test(m[0].slice(7)))),
+      },
+    ],
+  },
+  {
+    id: 'stripe-live-key',
+    lockfile: true,
+    description: 'Stripe live secret or restricted key',
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[0]),
+      },
+    ],
+  },
+  {
+    id: 'neon-api-key',
+    lockfile: true,
+    description: 'Neon API key (napi_ prefix)',
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[0]),
+      },
+    ],
+  },
+  {
+    id: 'neon-role-password',
+    lockfile: true,
+    description: 'Neon role password (npg_ prefix)',
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[0]),
+      },
+    ],
+  },
+  {
+    id: 'stack-auth-secret-key',
+    lockfile: true,
+    description: 'Stack Auth secret server key (ssk_ prefix)',
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[0]),
+      },
+    ],
+  },
+  {
+    id: 'jwt-token',
+    lockfile: true,
+    description: 'JWT-shaped token with a long signature',
+    matchers: [
+      {
+        pattern: /(? !placeholderOf(m[1]),
+      },
+    ],
+  },
+  ...PROVIDER_RULES,
+  LOCKFILE_RULE,
+  SQL_RULE,
+  {
+    id: 'webhook-url',
+    lockfile: true,
+    description:
+      'webhook URL whose path or query is a secret token (Slack, Discord, Microsoft Teams / Power Automate, Zapier, IFTTT, PagerDuty, Telegram bot): anyone holding the URL can post',
+    hint: /hooks\.slack|discord|webhook\.office|outlook\.office|logic\.azure|zapier|ifttt|api\.telegram|pagerduty/i,
+    matchers: [
+      {
+        // https://hooks.slack.com/services/T000/B000/, /workflows/T/A//, /triggers/E//
+        pattern: new RegExp(String.raw`hooks\.slack\.com${SLASH}(?:services|workflows|triggers)((?:${SLASH}[A-Za-z0-9_%-]{1,100}){1,6})`, 'gi'),
+        accept: (m) => {
+          const last = m[1].split(/\\?\//).pop();
+          return last.length >= 16 && !isPlaceholder(last);
+        },
+      },
+      {
+        // https://discord.com/api/webhooks//
+        pattern: new RegExp(String.raw`discord(?:app)?\.com${SLASH}api(?:${SLASH}v\d{1,2})?${SLASH}webhooks${SLASH}\d{5,25}${SLASH}([A-Za-z0-9_-]{16,})`, 'gi'),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+      {
+        // https://.webhook.office.com/webhookb2/@/IncomingWebhook/<32 hex>/  (and outlook.office.com/webhook/...)
+        pattern: new RegExp(
+          String.raw`(?:outlook\.office(?:365)?\.com${SLASH}webhook|[a-z0-9.-]{1,80}\.webhook\.office\.com${SLASH}webhook[a-z0-9]{0,3})${SLASH}[^\s"'<>]{0,300}?IncomingWebhook${SLASH}([A-Za-z0-9]{20,})`,
+          'gi',
+        ),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+      {
+        // Power Automate / Logic Apps HTTP trigger: https://prod-00.region.logic.azure.com/workflows//triggers/manual/paths/invoke?...&sig=
+        pattern: new RegExp(
+          String.raw`\.logic\.azure\.com(?::\d{1,5})?${SLASH}workflows${SLASH}[^\s"'<>]{0,300}?(?:[?&]|\\u0026|&)sig=([A-Za-z0-9_%-]{16,})`,
+          'gi',
+        ),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+      {
+        // https://hooks.zapier.com/hooks/catch//
+        pattern: new RegExp(String.raw`hooks\.zapier\.com${SLASH}hooks${SLASH}catch${SLASH}\d{3,}${SLASH}([A-Za-z0-9]{5,})`, 'gi'),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+      {
+        // https://maker.ifttt.com/trigger//with/key/
+        pattern: new RegExp(String.raw`maker\.ifttt\.com${SLASH}trigger${SLASH}[A-Za-z0-9_-]{1,100}${SLASH}(?:json${SLASH})?with${SLASH}key${SLASH}([A-Za-z0-9_-]{16,})`, 'gi'),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+      {
+        // https://events.pagerduty.com/integration/<32 character integration key>/enqueue
+        pattern: new RegExp(String.raw`events\.pagerduty\.com${SLASH}integration${SLASH}([A-Za-z0-9]{20,})${SLASH}enqueue`, 'gi'),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+      {
+        // https://api.telegram.org/bot:/sendMessage
+        pattern: new RegExp(String.raw`api\.telegram\.org${SLASH}bot(\d{6,}:[A-Za-z0-9_-]{30,})`, 'gi'),
+        accept: (m) => !placeholderOf(m[1]),
+      },
+    ],
+  },
+  {
+    id: 'secret-assignment',
+    description:
+      'secret-like name (SECRET, PASSWORD, TOKEN, API_KEY, ...) set to a non-placeholder literal: any 8+ character value in env/config files, a random-looking quoted literal in code',
+    hint: SECRET_HINT,
+    matchers: [
+      SECRET_ASSIGNMENT_MATCHER,
+      DEFINITION_ASSIGNMENT_MATCHER,
+      PROPERTIES_BLANK_MATCHER,
+      {
+        // Pulumi..yaml: `config:` keys are `:: value` (app:apiToken: V). The name is the part after the namespace; a
+        // secret stored by `pulumi config set --secret` is a `secure:` ciphertext mapping and passes.
+        appliesTo: (ctx) => ctx.mode === 'config' && /(?:^|\/)pulumi(?:\.[A-Za-z0-9_-]{1,64})?\.ya?ml$/i.test(ctx.path),
+        pattern: /(? {
+          // SECRET_ASSIGNMENT_MATCHER reads the name from group 2 and the separator from group 3.
+          const shifted = Object.assign([m[0], m[1], m[1], m[2]], { index: m.index, input: m.input });
+          const found = SECRET_ASSIGNMENT_MATCHER.accept(shifted, ctx);
+          if (shifted.spanEnd !== undefined) m.spanEnd = shifted.spanEnd;
+          return found;
+        },
+      },
+      {
+        // The value starts on a later line: YAML `NAME:` + an indented scalar, INI / configparser `NAME =` + indented
+        // continuation lines. A nested mapping (`NAME:` + `value: V`) is the name/value pair rule's business.
+        // group 2 = name. The name line ends the match, so the cost per start is the name plus a comment.
+        appliesTo: (ctx) => ctx.mode === 'config' && /\.(?:ya?ml|ini|cfg|conf)$/i.test(ctx.path),
+        pattern: /(? {
+          const kind = nameKindFor(m[2], ctx);
+          if (!kind) return false;
+          if (budgetSpent(ctx)) return exhaustedVerdict(m, ctx, { budget: true });
+          const input = m.input;
+          const yaml = /\.ya?ml$/i.test(ctx.path);
+          const lineStart = ctx.lineStart(m.index);
+          const keyIndent = /^[ \t]*(?:-[ \t]+)?/.exec(input.slice(lineStart, m.index + 1))[0].length;
+          const pieces = [];
+          let at = m.index + m[0].length;
+          let end = at;
+          for (let n = 0; n < 20 && at < input.length; n += 1) {
+            let next = input.indexOf('\n', at);
+            if (next === -1) next = input.length;
+            const line = input.slice(at, Math.min(next, at + 4096)).replace(/\r$/, '');
+            if (line.trim() !== '' && !/^[ \t]*[#;]/.test(line)) {
+              if (/^[ \t]*/.exec(line)[0].length <= keyIndent) break;
+              // A nested mapping or a list (`pass:` / `- "text"` in a test fixture) is not a scalar value of this name.
+              if (yaml && pieces.length === 0 && /^[ \t]*(?:[^\s#'"-][^:]*:(?:[ \t]|$)|-(?:[ \t]|$))/.test(line)) return false;
+              pieces.push(line.trim());
+              end = next;
+            }
+            at = next + 1;
+          }
+          if (pieces.length === 0 || isVaultBody(pieces)) return false;
+          if (!spendMultiline(ctx, end - m.index)) return exhaustedVerdict(m, ctx, { budget: true });
+          m.spanEnd = end;
+          const judge = (text) => isSecretValue({ kind, value: stripQuotes(text), quoted: true, separator: m[0].includes('=') ? '=' : ':', mode: ctx.mode, minLength: ctx.minStrong, catalog: ctx.catalog });
+          return pieces.some(judge) || (pieces.length > 1 && judge(pieces.join(' ')));
+        },
+      },
+      {
+        // Command-style assignments with no "=": fish `set -gx NAME value ...` (also csh `set NAME = value`, Windows `set "NAME=value"`),
+        // csh `setenv NAME value`, Windows `setx NAME value`, `export NAME value`. See the block above shellTail.
+        // group 1 = fish flags, 3 = name. The bounded flag list starts every flag with a blank and a "-", so it has one reading.
+        hint: /\b(?:set|setenv|setx|export)\b/i,
+        pattern: /(? {
+          const kind = nameKindFor(m[3], ctx);
+          if (!kind || !fishFlagsAssign(m[1]) || !commandContextOk(m, ctx, m[1] !== '')) return false;
+          const words = shellTail(m.input, m.index + m[0].length);
+          m.spanEnd = m.index + m[0].length;
+          return shellWordsAreSecret(kind, words, ctx);
+        },
+      },
+      {
+        // Windows: set "NAME=value" (the quotes keep spaces and & out of the operator syntax)
+        hint: /\bset[ \t]+"/i,
+        pattern: /(? {
+          const kind = nameKindFor(m[1], ctx);
+          if (!kind || !commandContextOk(m, ctx, false)) return false;
+          return shellWordsAreSecret(kind, [{ text: m[2], quoted: true, expr: false }], ctx);
+        },
+      },
+      {
+        // csh / tcsh: setenv NAME value
+        hint: /\bsetenv\b/i,
+        pattern: /(? {
+          const kind = nameKindFor(m[2], ctx);
+          if (!kind || !commandContextOk(m, ctx, true)) return false;
+          m.spanEnd = m.index + m[0].length;
+          return shellWordsAreSecret(kind, shellTail(m.input, m.index + m[0].length), ctx);
+        },
+      },
+      {
+        // Windows: setx [/M] NAME value [/M]
+        hint: /\bsetx\b/i,
+        pattern: /(? {
+          const kind = nameKindFor(m[2], ctx);
+          if (!kind || !commandContextOk(m, ctx, true)) return false;
+          m.spanEnd = m.index + m[0].length;
+          return shellWordsAreSecret(kind, shellTail(m.input, m.index + m[0].length), ctx);
+        },
+      },
+      {
+        // sh: export NAME value (no "="). A list of plain variable names (`export A B`) is not a value.
+        hint: /\bexport\b/,
+        pattern: /(? {
+          const kind = nameKindFor(m[3], ctx);
+          if (!kind || !commandContextOk(m, ctx, false)) return false;
+          const words = shellTail(m.input, m.index + m[0].length).filter((w) => w.quoted || w.text !== '=');
+          if (words.length === 0 || words.every((w) => !w.quoted && SHELL_VARIABLE_NAME.test(w.text))) return false;
+          m.spanEnd = m.index + m[0].length;
+          return shellWordsAreSecret(kind, words, ctx);
+        },
+      },
+      {
+        // PowerShell: $env:NAME = 'value', ${env:NAME} += "value"
+        hint: /env:/i,
+        pattern: /\$\{?env:([A-Za-z_][A-Za-z0-9_.-]{0,255})\}?[ \t]{0,64}\+?=[ \t]{0,64}(?=["'])/gi,
+        accept: (m, ctx) => {
+          const kind = nameKindFor(m[1], ctx);
+          if (!kind || !commandContextOk(m, ctx, true)) return false;
+          const literal = powershellString(m.input, m.index + m[0].length);
+          if (literal !== null) m.spanEnd = literal.end;
+          return powershellValueIsSecret(kind, literal, ctx);
+        },
+      },
+      {
+        // PowerShell / .NET: [Environment]::SetEnvironmentVariable('NAME', 'value'[, 'User'])
+        hint: /SetEnvironmentVariable/i,
+        pattern: /\[(?:System\.)?Environment\][ \t]{0,64}::[ \t]{0,64}SetEnvironmentVariable\([ \t]{0,64}(["'])([A-Za-z_][A-Za-z0-9_.-]{0,255})\1[ \t]{0,64},[ \t]{0,64}(?=["'])/gi,
+        accept: (m, ctx) => {
+          const kind = nameKindFor(m[2], ctx);
+          if (!kind || !commandContextOk(m, ctx, true)) return false;
+          const literal = powershellString(m.input, m.index + m[0].length);
+          if (literal !== null) m.spanEnd = literal.end;
+          return powershellValueIsSecret(kind, literal, ctx);
+        },
+      },
+      {
+        // Function-call setters (see ENV_SETTER_CALLEE): the second argument is a literal, or the name is followed by `=` in one string.
+        pattern: ENV_SETTER_CALL,
+        accept: (m, ctx) => {
+          if (ctx.mode === 'prose' && ctx.fenceLang(m.index) === undefined) return false;
+          const kind = nameKindFor(m[2], ctx);
+          if (!kind) return false;
+          const start = m.index + m[0].length;
+          let literal;
+          if (m[3] === '=') {
+            // putenv("NAME=value"): the value runs to the closing quote of the same string
+            const body = callStringLiteral(m.input, m.index + m[0].lastIndexOf('(') + 1 + /^[ \t]*/.exec(m[0].slice(m[0].lastIndexOf('(') + 1))[0].length);
+            literal = body === null ? null : { text: body.text.slice(body.text.indexOf('=') + 1), end: body.end, interpolated: body.interpolated };
+          } else {
+            literal = callStringLiteral(m.input, start);
+          }
+          if (literal === null) return false;
+          m.spanEnd = literal.end;
+          if (literal.interpolated && /[{}]/.test(literal.text)) return false; // an interpolated string is built from other values
+          return powershellValueIsSecret(kind, { text: literal.text }, ctx);
+        },
+      },
+      {
+        // Perl hash elements: $ENV{NAME} = 'v', $config{'password'} = "v" (a subscript in braces; PHP and Ruby use brackets, read above)
+        pattern: /(? {
+          if (ctx.mode === 'prose' && ctx.fenceLang(m.index) === undefined) return false;
+          const kind = nameKindFor(m[2], ctx);
+          if (!kind) return false;
+          const literal = callStringLiteral(m.input, m.index + m[0].length);
+          if (literal === null) return false;
+          m.spanEnd = literal.end;
+          return powershellValueIsSecret(kind, { text: literal.text }, ctx);
+        },
+      },
+      {
+        // PowerShell: Set-Item -Path Env:NAME -Value 'v', Set-Item Env:\NAME 'v', New-Item -Path Env:NAME -Value 'v'
+        hint: /Env:/i,
+        pattern: /(? {
+          const kind = nameKindFor(m[1], ctx);
+          if (!kind || !commandContextOk(m, ctx, true)) return false;
+          const words = shellTail(m.input, m.index);
+          const at = words.findIndex((w) => !w.quoted && /^-value$/i.test(w.text));
+          const rest = words.slice(1);
+          const value = at !== -1 ? words[at + 1] : rest.find((w) => w.quoted || !/^-|^Env:/i.test(w.text) && w.text !== '=');
+          m.spanEnd = m.index + m[0].length;
+          return value !== undefined && shellWordsAreSecret(kind, [value], ctx);
+        },
+      },
+      {
+        // fish universal variables file (~/.config/fish/fish_variables): SETUVAR [--export] NAME:value
+        hint: /SETUVAR/,
+        pattern: /^SETUVAR((?:[ \t]{1,64}--?[A-Za-z-]{1,20}){0,4})[ \t]{1,64}([A-Za-z_][A-Za-z0-9_]{0,255}):([^\n]{1,4096})$/gm,
+        accept: (m, ctx) => {
+          const kind = nameKindFor(m[2], ctx);
+          if (!kind) return false;
+          const value = m[3].replace(/\\x1[de]/gi, ' ').replace(/\\(.)/g, '$1').replace(/\r$/, '').trim();
+          return isSecretValue({ kind, value, quoted: true, separator: '=', mode: 'config', minLength: ctx.minStrong, catalog: ctx.catalog });
+        },
+      },
+      {
+        // Dockerfile "ENV NAME value" / "ARG NAME value" (space-separated; NAME=value is handled above)
+        pattern: /^[ \t]*(?:ONBUILD[ \t]+)?(?:ENV|ARG)[ \t]+([A-Za-z_][A-Za-z0-9_]{0,1023})[ \t]+([^\n]{1,4096})$/gim,
+        accept: (m, ctx) => {
+          if (!/(?:^|\/)(?:[^/]*dockerfile[^/]*|containerfile[^/]*)$/i.test(ctx.path)) return false;
+          const kind = secretNameKind(m[1]);
+          if (!kind || m[2].startsWith('=')) return false;
+          return isSecretValue({ kind, value: stripQuotes(m[2].replace(/\r$/, '')), quoted: true, separator: '=', mode: ctx.mode, catalog: ctx.catalog });
+        },
+      },
+      {
+        // XML element whose NAME is the secret-like word: Maven V, V,
+        // V, V, a prefixed WS-Security V,
+        // or an entry named by an attribute: V, V. XML configuration
+        // files only (an HTML page or SVG has  markup, not settings). The text is a bounded [^<>] run and the closing
+        // tag must repeat the name, so each start costs at most the text length and the scan stays linear.
+        // group 1 = qualified element name, 2 = start-tag attributes, 3 = text
+        hint: /<\//,
+        appliesTo: (ctx) => ctx.xml,
+        pattern: /<((?:[A-Za-z_][A-Za-z0-9_.-]{0,50}:)?[A-Za-z_][A-Za-z0-9_.-]{0,100})((?:[ \t\r\n][^<>]{0,300})?)>([^<>]{1,4096})<\/\1[ \t\r\n]*>/g,
+        accept: (m, ctx) => xmlElementIsSecret(m[1], m[2], m[3], ctx),
+      },
+      {
+        // The same element with its text in a CDATA section: . The section body is a run of
+        // non-"]" characters and "]" not followed by "]>", so every character has one reading and the cost per start is bounded.
+        hint: / ctx.xml,
+        pattern: /<((?:[A-Za-z_][A-Za-z0-9_.-]{0,50}:)?[A-Za-z_][A-Za-z0-9_.-]{0,100})((?:[ \t\r\n][^<>]{0,300})?)>[ \t\r\n]{0,64})){1,1024})\]\]>[ \t\r\n]{0,64}<\/\1[ \t\r\n]*>/g,
+        accept: (m, ctx) => xmlElementIsSecret(m[1], m[2], m[3], ctx),
+      },
+    ],
+  },
+  {
+    id: 'secret-name-value-pair',
+    description:
+      'name/value pair split across fields, in any order and with other fields in between (k8s "- name: X / value: Y", Vercel {"key":"X","value":"Y"}, {name: X, value: Y}, Terraform blocks, XML , CloudFormation ParameterKey/ParameterValue, create_var(name=X, value=Y), nested "X: {value: Y}") where the name is secret-like',
+    hint: SECRET_HINT,
+    matchers: [
+      {
+        appliesTo: (ctx) => ctx.mode !== 'code',
+        // group 3 = variable name, 5/6 = quoted value, 7 = bare value
+        pattern:
+          /(? {
+          const kind = secretNameKind(m[3]);
+          if (!kind) return false;
+          const quotedValue = m[4] ?? m[5];
+          const quoted = quotedValue !== undefined;
+          return isSecretValue({
+            kind,
+            value: quoted ? unescapeQuoted(quotedValue) : m[6],
+            quoted,
+            separator: ':',
+            mode: ctx.mode,
+            catalog: ctx.catalog,
+          });
+        },
+      },
+      {
+        // The same pair when the fields are not adjacent or not in that order: {"value":"Y","key":"X"},
+        // {"key":"X","type":"encrypted","value":"Y"}, {name: X, value: Y}, `- value: Y` above `name: X`, Terraform
+        // `name = "X"` / `value = "Y"` blocks, Netlify {"key":"X","values":[{"value":"Y"}]}. The value is searched for
+        // in the enclosing bounded { ... } object and in the neighbouring lines of the YAML mapping.
+        // group 1 = quote (may be a backslash-escaped one), 3 = variable name
+        pattern:
+          /(? {
+          // In source code the name must be a quoted literal ('JWT_SECRET'); a bare identifier (`key = TOKEN`) is code, not data.
+          if (ctx.mode === 'code' && m[2] === '') return false;
+          const kind = nameKindFor(m[3], ctx);
+          return kind !== null && pairValueIsSecret(m, kind, ctx);
+        },
+      },
+      {
+        // XML element form: XY, X...
+        hint: /<\/(?:name|key|variable|env)/i,
+        pattern: /<(name|key|variable|env|parametername|parameterkey)>[ \t\r\n]*([A-Za-z_][A-Za-z0-9_.-]{0,1023})[ \t\r\n]*<\/\1>/gi,
+        accept: (m, ctx) => {
+          const kind = nameKindFor(m[2], ctx);
+          if (kind === null) return false;
+          const window = xmlWindow(m.input, m.index);
+          ctx.pairBudget = (ctx.pairBudget ?? PAIR_BUDGET_CHARS) - (window?.text.length ?? 0) - 64;
+          if (ctx.pairBudget < 0) {
+            ctx.pairExhausted = true;
+            attributeToFile(m);
+            return true;
+          }
+          const found = window === null ? null : windowHoldsSecretValue(window, kind, ctx, false, m.input);
+          return found !== null && pairHit(m, window, found, ctx);
+        },
+      },
+      {
+        // The name is the KEY of a mapping whose child holds the value: `secrets:\n  NAME:\n    value: Y`, {"NAME": {"value": "Y"}},
+        // `NAME: {value: Y}`. group 2 = quote, 3 = name; the child is the indented block below, or the object that follows.
+        pattern:
+          /(? {
+          if (ctx.mode === 'code' && m[1] === '') return false;
+          const kind = nameKindFor(m[2], ctx);
+          if (kind === null) return false;
+          const start = ctx.lineStart(m.index);
+          const prefix = m.input.slice(start, m.index);
+          const isBlock = m.input[m.index + m[0].length] !== '{';
+          if (isBlock && (ctx.mode === 'code' || !/^[ \t]*(?:-[ \t]+)*$/.test(prefix))) return false;
+          return childHoldsSecretValue(m, kind, ctx, prefix.length);
+        },
+      },
+      {
+        // Terraform / OpenTofu / Packer blocks whose secret-like LABEL is the variable's name: variable "api_token" { default = "V" },
+        // output "api_token" { value = "V" }. The label is neither a name/key field nor an assignment, so the matchers above never pair it
+        // with `default` / `value`. A `sensitive = true` or `type = string` alone carries no value and passes.
+        hint: /(?:variable|output)[ \t]/,
+        pattern: /(? ctx.mode !== 'code',
+        accept: (m, ctx) => {
+          const kind = nameKindFor(m[2], ctx);
+          return kind !== null && childHoldsSecretValue(m, kind, ctx, 0);
+        },
+      },
+      {
+        // Delimited rows: `NAME,value` (CSV), `NAMEvalue` (TSV), `| NAME | value |` (Markdown or text tables).
+        pattern:
+          /^[ \t]*\|?[ \t]*(["'`]?)([A-Za-z_][A-Za-z0-9_.-]{0,1023})\1[ \t]*([,\t|;])[ \t]*(?:"([^"\n]{0,4096})"|`([^`\n]{0,4096})`|([^,\t|;\n"`]{1,4096}))/gm,
+        accept: (m, ctx) => {
+          const table = /\.(?:csv|tsv|psv|tab)$/i.test(ctx.path);
+          const prose = ctx.mode === 'prose';
+          if (!table && !(prose && m[3] === '|')) return false;
+          const kind = secretNameKind(m[2]);
+          if (kind === null) return false;
+          const quotedValue = m[4] ?? m[5];
+          const quoted = quotedValue !== undefined;
+          return isSecretValue({
+            kind,
+            value: (quoted ? quotedValue : m[6]).trim(),
+            quoted,
+            separator: table ? '=' : ':',
+            mode: table ? 'config' : 'prose',
+            catalog: ctx.catalog,
+          });
+        },
+      },
+    ],
+  },
+  {
+    id: 'secret-cli-command',
+    description:
+      'secret-like variable set on a command line with a literal value (gh secret set NAME --body V, netlify env:set NAME V, vercel env add NAME <<< V, aws ssm put-parameter --name NAME --value V, kubectl create secret --from-literal)',
+    hint: /\b(?:gh|netlify|vercel|heroku|fly|flyctl|wrangler|railway|doppler|aws|firebase|az|kubectl|docker|pulumi)[ \t]/,
+    matchers: [
+      {
+        pattern:
+          /(? {
+          const tool = CLI_TOOLS.find(([re]) => re.test(m[0]));
+          if (!tool) return false;
+          // `echo V | vercel env add NAME`, `cat < secretNameKind(name) ?? (tool[1].secretFlag !== undefined && words.includes(tool[1].secretFlag) ? 'strong' : null);
+          const judge = (piped) =>
+            cliPairs(words, options, piped).some(([name, value]) => {
+              const kind = nameKind(name);
+              return kind !== null && isSecretValue({ kind, value: value.trim(), quoted: true, separator: '=', mode: 'config', catalog: false });
+            });
+          const found = (stdin.values.length === 0 ? [null] : stdin.values).some(judge);
+          // A heredoc whose delimiter is not found within the bounds is not verified: fail closed for a secret-like name.
+          const unverified = !found && stdin.unterminated && cliPairs(words, options, HEREDOC_UNVERIFIED).some(([name, value]) => value === HEREDOC_UNVERIFIED && nameKind(name) !== null);
+          if (found || unverified) {
+            // The heredoc body is part of the match: an allow marker on one of its lines counts, and so does a change to one of them.
+            if (stdin.end > m.index + m[0].length) m.spanEnd = stdin.end;
+            return true;
+          }
+          return false;
+        },
+      },
+    ],
+  },
+  {
+    id: 'credential-file',
+    description:
+      'secret in a native credential-file format: .netrc, .pgpass, .git-credentials, .npmrc/.yarnrc, .htpasswd, .curlrc, .vault-token, Kubernetes client-key-data and .dockerconfigjson',
+    matchers: [
+      {
+        // .netrc / _netrc: `machine H login U password P`, `default login U password P`, `account A`; tokens may span lines.
+        appliesTo: hasFormat('netrc'),
+        pattern: /(? {
+          const value = unescapeQuoted(m[1] ?? m[2]);
+          return !onCommentLine(m, ctx) && !isAnonymousFtpValue(value) && credentialValueIsSecret(value);
+        },
+      },
+      {
+        // .pgpass: hostname:port:database:username:password (a backslash escapes ":" and "\"). Comment lines start with #.
+        appliesTo: hasFormat('pgpass'),
+        pattern:
+          /^(?![ \t]*#)[ \t]*(?:[^:\\\n]|\\.){0,255}:(?:[^:\\\n]|\\.){0,255}:(?:[^:\\\n]|\\.){0,255}:(?:[^:\\\n]|\\.){0,255}:([^\n]+)$/gm,
+        accept: (m) => credentialValueIsSecret(unescapeQuoted(m[1])),
+      },
+      {
+        // .git-credentials: https://user:password@host lines. url-password judges those on real hosts; this covers
+        // documentation hosts (a stored credential is a credential wherever it points) and a token used as the user name.
+        appliesTo: hasFormat('gitcred'),
+        pattern: /^[ \t]*[a-z][a-z0-9+.-]*:\/\/([^\s:@/]{1,1024})(?::([^\s@/]{1,4096}))?@([^\s/?#]{0,256})/gim,
+        accept: (m) => {
+          if (m[2] === undefined) {
+            const user = safeDecode(m[1]);
+            return user.length >= 8 && !isPlaceholder(user) && !isWordIdentifier(user);
+          }
+          return isDocumentationHost(m[3]) && urlPasswordIsSecret(m[2]);
+        },
+      },
+      {
+        // .npmrc / .yarnrc in the space-separated form: "//registry.example/:_authToken" "value", npmAuthIdent "u:p"
+        appliesTo: hasFormat('npmrc'),
+        pattern:
+          /(? credentialValueIsSecret(unescapeQuoted(m[1] ?? m[2] ?? m[3])),
+      },
+      {
+        // .htpasswd / .htdigest: user:hash. The hash is offline-crackable, so a committed one counts as a credential.
+        appliesTo: hasFormat('htpasswd'),
+        pattern: /^(?![ \t]*#)[ \t]*[^\s:#][^:\n]{0,255}:(?:[^:\n]{0,255}:)?([^\s:]{1,4096})[ \t]*\r?$/gm,
+        // A hash is a credential; a placeholder body ($apr1$xxxxxxxx$xxxxxxxxxxxxxxxxxxxxxx, {SHA}REDACTED) is documentation.
+        accept: (m) => {
+          const hashBody = /^\$|^\{[A-Za-z0-9]+\}/.test(m[1]) ? m[1].slice(Math.max(m[1].lastIndexOf('$'), m[1].lastIndexOf('}')) + 1) : null;
+          return hashBody !== null ? hashBody.length >= 4 && !isPlaceholder(hashBody) : credentialValueIsSecret(m[1]);
+        },
+      },
+      {
+        // .curlrc: user = "name:password"  (also -u / --user)
+        appliesTo: hasFormat('curlrc'),
+        pattern: /^[ \t]*(?:-u|--user|user)[ \t]*(?:=|[ \t])[ \t]*(?:"((?:[^"\\\n]|\\.){0,1024})"|'([^'\n]{0,1024})'|(\S{1,1024}))/gm,
+        accept: (m) => {
+          const userAndPassword = unescapeQuoted(m[1] ?? m[2] ?? m[3]);
+          const colon = userAndPassword.indexOf(':');
+          return colon !== -1 && credentialValueIsSecret(userAndPassword.slice(colon + 1));
+        },
+      },
+      {
+        // .vault-token: the whole file is the token
+        appliesTo: hasFormat('vault'),
+        pattern: /^[ \t]*([^\s#][^\n]{0,4095})$/gm,
+        accept: (m) => m[1].trim().length >= MIN_STRONG_CONFIG_LENGTH && credentialValueIsSecret(m[1]),
+      },
+      {
+        // Kubernetes kubeconfig and image-pull secrets, in any file name: client-key-data, .dockerconfigjson, .dockercfg
+        appliesTo: (ctx) => ctx.mode !== 'code',
+        hint: /client-key-data|\.dockerconfigjson|\.dockercfg/,
+        pattern:
+          /(? {
+          const value = m[2] ?? m[3] ?? m[4];
+          return value.length >= 20 && !isPlaceholder(value);
+        },
+      },
+      // ---- Content matchers: the same credential files under any name (netrc.txt, notes.md, a script or CI step that
+      // writes the file with echo/printf/heredoc). They need no path, and skip the path-specific format that already covers them.
+      {
+        // machine H login U password P / default login U password P / machine H password P (host with a dot), across lines
+        // or with literal \n separators (printf "machine h\nlogin u\npassword p\n").
+        hint: /machine|default/,
+        pattern: new RegExp(
+          String.raw`(? {
+          if (ctx.formats.has('netrc')) return false;
+          const hasLogin = /login|user/.test(m[2]);
+          if (!hasLogin && !(m[1] ?? '').includes('.')) return false; // "machine learning password reset" is prose
+          const value = unescapeQuoted(m[3] ?? m[4]);
+          return !isAnonymousFtpValue(value) && credentialValueIsSecret(value);
+        },
+      },
+      {
+        // .pgpass line, anywhere: host:port:database:user:password with a numeric port, alone on its line or written by
+        // echo/printf (`echo "db:5432:d:u:pw" > ~/.pgpass`). Anchored to the start of the line so it stays linear.
+        pattern:
+          /^[ \t]*(?:-[ \t]+)*(?:[\w-]+:[ \t]+)?(?:(?:echo|printf)(?:[ \t]+-[A-Za-z]+)*[ \t]+)?["']?([A-Za-z0-9_.*-]{1,255}):(?:\d{2,5}|\*):[A-Za-z0-9_.*-]{1,255}:([A-Za-z0-9_.@*-]{1,255}):([^\s:"'<>|;&]{4,1024})(?=["']?[ \t]*(?:>|\r?$))/gm,
+        accept: (m, ctx) => !ctx.formats.has('pgpass') && /[A-Za-z*]/.test(m[1]) && /[A-Za-z]/.test(m[2]) && credentialValueIsSecret(m[3]),
+      },
+      {
+        // Docker registry auth in any JSON/YAML/echo: {"auths": {"host": {"auth": ""}}}
+        hint: /auths/,
+        pattern:
+          /(? {
+          if (ctx.formats.has('docker') || !ctx.hasBefore('auths', m.index, 500)) return false; // registry entries sit right under "auths"
+          return credentialValueIsSecret(m[3] ?? m[4] ?? m[5]);
+        },
+      },
+    ],
+  },
+  {
+    id: 'http-auth-credential',
+    description:
+      'HTTP credential written out: Authorization / Proxy-Authorization with a Bearer, Basic, token or ApiKey value (header text, curl -H, headers.set(...), object or YAML or JSON fields, nginx / Apache / HAProxy header directives), and Basic-auth calls with a literal password (auth=("user", "pw"), HTTPBasicAuth, Credentials.basic)',
+    hint: /auth|basic|credential/i,
+    matchers: [
+      {
+        // Authorization: Bearer   |  "Authorization": "Basic "  |  .set('Authorization', 'token ')
+        pattern: new RegExp(HTTP_AUTH_NAME + HTTP_AUTH_SEPARATOR + HTTP_CREDENTIAL, 'gi'),
+        accept: (m, ctx) => {
+          if ((m[2] ?? '').toLowerCase() === 'digest') return false; // judged by its own matcher
+          // In source code a bare word after the name is an identifier (`Authorization: authHeader`), not a literal token.
+          if (ctx.mode === 'code' && m[2] === undefined && m[1] === '') return false;
+          return httpCredentialIsSecret(m[2], m[3]);
+        },
+      },
+      {
+        // proxy_set_header Authorization "Bearer ";  RequestHeader set Authorization "Basic "  http-request set-header Authorization ...
+        pattern: new RegExp(HTTP_HEADER_DIRECTIVE + String.raw`[ \t]+["']?(?:[A-Za-z]{1,20}-)?authorization["']?[ \t]{1,8}` + HTTP_CREDENTIAL, 'gi'),
+        accept: (m, ctx) => directiveContextOk(m, ctx) && (m[2] ?? '').toLowerCase() !== 'digest' && httpCredentialIsSecret(m[2], m[3]),
+      },
+      {
+        // Authorization: Digest username="u", realm="r", nonce="n", uri="/", response=""  (the response is replayable within its nonce)
+        pattern: new RegExp(HTTP_AUTH_NAME + HTTP_AUTH_SEPARATOR + String.raw`["'\x60]?digest[ \t][^\n]{0,600}?\bresponse=\\?["']?([A-Za-z0-9+/=_-]{16,256})`, 'gi'),
+        accept: (m) => !isPlaceholder(m[1]) && looksRandom(m[1], { minLength: 16, minEntropy: 3.0 }),
+      },
+      {
+        // requests: auth=("user", "pw"), HTTPBasicAuth('user', 'pw'); Java / Kotlin / C# / Go: Credentials.basic("u", "pw"), new UsernamePasswordCredentials("u", "pw"),
+        // new Basic("u", "pw"), NetworkCredential, SetBasicAuth; supertest .auth('u', 'pw'). The second argument is a literal password.
+        hint: /auth|basic|credential/i,
+        // The pattern finds the call and its first argument; the second argument, one complete quoted string ("""triple""", 'single',
+        // "double" or `backtick`, escapes honoured, spaces allowed: a passphrase), is read by quotedArgument, one forward scan with no
+        // backtracking, so no regex has to tell a quote that closes the string from one inside it.
+        pattern:
+          /(?:\.auth\(|(? {
+          const argument = quotedArgument(m.input, m.index + m[0].length);
+          if (argument === null || argument.raw.length === 0) return false;
+          m.spanEnd = argument.end;
+          return isSecretValue({ kind: 'strong', value: unescapeQuoted(argument.raw), quoted: true, separator: '=', mode: 'config' });
+        },
+      },
+    ],
+  },
+  {
+    id: 'config-directive-secret',
+    description:
+      'password on a whitespace-delimited service directive: Redis requirepass / masterauth / sentinel auth-pass / ACL >password, Mosquitto, HAProxy userlist, msmtp, fetchmail, nginx / Apache header and variable directives, ssl_passphrase_command, OpenVPN inline credentials',
+    hint: /secret|passw|pwd|pass|token|credential|key|auth|user[ \t]|header/i,
+    matchers: [
+      {
+        // redis.conf: requirepass P, masterauth P, tls-key-file-pass P, tls-client-key-file-pass P; and the same as command-line options
+        // (redis-server --requirepass P) or commands (CONFIG SET requirepass P). group 1 is set when the directive starts its line.
+        pattern: new RegExp(
+          String.raw`(?:^([ \t]*)|--|\bset[ \t]+)(?:requirepass|masterauth|tls-key-file-pass|tls-client-key-file-pass)(?:[ \t]+|=|["'][ \t]{0,4},[ \t]{0,4}["'])${DIRECTIVE_VALUE}`,
+          'gim',
+        ),
+        accept: (m, ctx) => (m[1] === undefined || directiveContextOk(m, ctx)) && directiveIsSecret(directiveValue(m.groups)),
+      },
+      {
+        // sentinel.conf: sentinel auth-pass  P, sentinel requirepass P
+        pattern: new RegExp(String.raw`^[ \t]*sentinel[ \t]+(?:auth-pass[ \t]+\S{1,256}|requirepass)[ \t]+${DIRECTIVE_VALUE}`, 'gim'),
+        accept: (m, ctx) => directiveContextOk(m, ctx) && directiveIsSecret(directiveValue(m.groups)),
+      },
+      {
+        // ACL rules: `user default on >P ~* +@all`, `user app on # ...`, ACL SETUSER app on >P
+        pattern: /^[ \t]*(?:user|ACL[ \t]+SETUSER)[ \t]+\S{1,256}[ \t]([^\n]{1,2000})$/gim,
+        accept: (m, ctx) => directiveContextOk(m, ctx) && aclRulesHoldSecret(m[1]),
+      },
+      {
+        // Files of a service that reads `name value`: redis, mosquitto (password P, bridge_password P, remote_password P), msmtp (password P),
+        // tinyproxy. The NAME must be secret-like (nameKindFor), so `passwordeval`, `password_file` and `requirepass`-free lines pass.
+        pattern: new RegExp(String.raw`^[ \t]*(?[A-Za-z][A-Za-z0-9_.-]{0,127})[ \t]+${DIRECTIVE_VALUE}`, 'gm'),
+        appliesTo: (ctx) => ctx.mode === 'config' && serviceConfigOf(ctx.path) !== null && serviceConfigOf(ctx.path) !== 'fetchmail',
+        accept: (m, ctx) => {
+          if (/^(?:BasicAuth|basicauth)$/.test(m.groups.name)) return false;
+          const kind = nameKindFor(m.groups.name, ctx);
+          const value = directiveValue(m.groups);
+          // Only names that end in a secret noun: a weak name (`password_file`, `X-Vault-Key`) usually holds a path or an id.
+          return kind === 'strong' && directiveIsSecret(value);
+        },
+      },
+      {
+        // tinyproxy.conf: BasicAuth  
+        pattern: new RegExp(String.raw`^[ \t]*BasicAuth[ \t]+\S{1,256}[ \t]+${DIRECTIVE_VALUE}`, 'gim'),
+        appliesTo: (ctx) => ctx.mode === 'config' && serviceConfigOf(ctx.path) === 'tinyproxy',
+        accept: (m) => directiveIsSecret(directiveValue(m.groups)),
+      },
+      {
+        // .fetchmailrc: poll host protocol pop3 user "alice" password "P"  (also `with password P`)
+        pattern: new RegExp(String.raw`(? ctx.mode === 'config' && serviceConfigOf(ctx.path) === 'fetchmail',
+        accept: (m) => directiveIsSecret(directiveValue(m.groups)),
+      },
+      {
+        // HAProxy userlist: user NAME [groups G] password HASH | insecure-password P   (a crypt(3) hash after `password` is not plaintext)
+        pattern: new RegExp(String.raw`^[ \t]*user[ \t]+\S{1,256}[ \t]+(?:groups[ \t]+\S{1,256}[ \t]+)?(?insecure-password|password)[ \t]+${DIRECTIVE_VALUE}`, 'gim'),
+        appliesTo: (ctx) => ctx.mode === 'config',
+        accept: (m, ctx) => {
+          if (!ctx.hasBefore('userlist', m.index, 8000)) return false;
+          const value = directiveValue(m.groups);
+          if (m.groups.kind.toLowerCase() === 'password' && /^\$\d[a-z]?\$/.test(value.text)) return false;
+          return directiveIsSecret(value);
+        },
+      },
+      {
+        // HAProxy: stats auth USER:PASSWORD
+        pattern: /^[ \t]*stats[ \t]+auth[ \t]+[^\s:]{1,256}:(\S{1,4096})/gim,
+        appliesTo: (ctx) => ctx.mode === 'config',
+        accept: (m) => directiveIsSecret({ text: m[1], quoted: false }),
+      },
+      {
+        // nginx, Apache and HAProxy directives that set a header, a FastCGI / uwsgi parameter or a variable to a literal under a secret-like NAME:
+        // proxy_set_header X-Api-Key "V";  fastcgi_param DB_PASSWORD V;  set $api_token "V";  SetEnv DB_PASSWORD V;  RequestHeader set X-Api-Key V
+        pattern: new RegExp(
+          String.raw`(?\$?[A-Za-z_][A-Za-z0-9_.-]{0,127})["']?[ \t]+${DIRECTIVE_VALUE}`,
+          'gi',
+        ),
+        accept: (m, ctx) => {
+          if (!directiveContextOk(m, ctx)) return false;
+          // The bare `set NAME value` is nginx only with a `$variable` (fish and csh also have `set`).
+          if (/^set[ \t]/i.test(m[0]) && !m.groups.name.startsWith('$')) return false;
+          const kind = nameKindFor(m.groups.name, ctx);
+          const value = directiveValue(m.groups);
+          // Only names that end in a secret noun: a weak name (`password_file`, `X-Vault-Key`) usually holds a path or an id.
+          return kind === 'strong' && directiveIsSecret(value);
+        },
+      },
+      {
+        // postgresql.conf: ssl_passphrase_command = 'echo P'  (the passphrase written into the command)
+        pattern: /(? ctx.mode === 'config',
+        accept: (m) => {
+          const echo = /(?:^|[ \t;|&])(?:echo|printf)[ \t]+(?:-[A-Za-z]+[ \t]+)*(?:"([^"\n]{1,512})"|''([^'\n]{1,512})''|([^\s"'|;&]{1,512}))/.exec(m[1]);
+          return echo !== null && directiveIsSecret({ text: echo[1] ?? echo[2] ?? echo[3], quoted: false });
+        },
+      },
+      {
+        // Erlang application config (RabbitMQ advanced.config / rabbitmq.config, sys.config): {default_pass, <<"P">>}, {password, "P"}
+        pattern: /\{[ \t\r\n]{0,16}(?[a-z][A-Za-z0-9_@]{0,127})[ \t\r\n]{0,16},[ \t\r\n]{0,16}(?:<<[ \t]{0,4})?"(?(?:[^"\\\n]|\\.){0,4096})"/g,
+        appliesTo: (ctx) => ctx.mode === 'config' && /\.config(?:\.src)?$|\.app\.src$/i.test(ctx.path),
+        accept: (m, ctx) => nameKindFor(m.groups.name, ctx) === 'strong' && directiveIsSecret({ text: unescapeQuoted(m.groups.dq), quoted: true }),
+      },
+      {
+        // OpenVPN inline credentials:  / user / password / 
+        hint: //,
+        pattern: /[ \t]*\r?\n[^\n]{0,256}\n([^\n<]{1,1024})\n[ \t]*<\/auth-user-pass>/g,
+        accept: (m) => directiveIsSecret({ text: m[1].trim(), quoted: false }),
+      },
+      {
+        // echo 'user:P' | chpasswd, chpasswd <<< "user:P": the password of a user account set from a script (a Dockerfile RUN line)
+        hint: /chpasswd/,
+        pattern: /(?:(?:echo|printf)[ \t]+(?:-[A-Za-z]+[ \t]+)*["']?[A-Za-z_][A-Za-z0-9_.-]{0,63}:([^\s"'|;&]{1,1024})["']?[ \t]*\|[ \t]*(?:sudo[ \t]+)?chpasswd|chpasswd[ \t]*<<<[ \t]*["']?[A-Za-z_][A-Za-z0-9_.-]{0,63}:([^\s"'|;&]{1,1024}))/g,
+        accept: (m) => {
+          const password = m[1] ?? m[2];
+          return !/^\$/.test(password) && directiveIsSecret({ text: password, quoted: false });
+        },
+      },
+    ],
+  },
+  {
+    id: 'hardcoded-signing-key',
+    description: 'jwt.sign / jwt.verify called with a random-looking string literal as the key',
+    appliesTo: (ctx) => ctx.mode === 'code',
+    hint: /\b(?:sign|verify)\b/,
+    matchers: [
+      {
+        pattern: /\b(?:jwt|jsonwebtoken|jws)\.(?:sign|verify)\(([^;]{0,300}?),[ \t]*(["'`])([^"'`\n]{8,4096})\2/g,
+        accept: (m) => isSecondArgument(m[1]) && looksRandom(m[3], GATES.codeStrong),
+      },
+    ],
+  },
+  {
+    id: 'hardcoded-secret-fallback',
+    description:
+      'process.env. || "" in code, including env["X"], destructuring defaults and os.getenv("X", "") (a hardcoded default secret)',
+    appliesTo: (ctx) => ctx.mode === 'code',
+    hint: SECRET_HINT,
+    matchers: [
+      {
+        // group 1 = .NAME, 3 = ["NAME"], 5 = literal
+        pattern: new RegExp(
+          String.raw`${ENV_ROOT}(?:\.([A-Za-z_$][A-Za-z0-9_$]{0,1023})|\[\s*(["'\x60])([^"'\x60\]\n]{1,1023})\2\s*\])\s*(?:\|\||\?\?)\s*` +
+            String.raw`(["'\x60])([^"'\x60\n]{1,4096})\4`,
+          'g',
+        ),
+        accept: (m) => isSecretLikeName(m[1] ?? m[3]) && !m[5].includes('${'),
+      },
+      {
+        // destructuring defaults: const { NAME = "x" } = process.env, with a secret-like NAME
+        pattern: new RegExp(String.raw`\{([^{}]{1,500})\}\s*=\s*${ENV_ROOT}\b`, 'g'),
+        accept: (m) => {
+          const entry = /(? isSecretLikeName(e[1]) && !e[3].includes('${'));
+        },
+      },
+      {
+        // Python defaults: os.getenv("NAME", "x") and os.environ.get("NAME", "x"), with a secret-like NAME
+        pattern: /\b(?:os\.environ\.get|os\.getenv|environ\.get|getenv)\(\s*(["'])([A-Za-z0-9_]{1,1023})\1\s*,\s*(["'])([^"'\n]{1,4096})\3/g,
+        accept: (m) => isSecretLikeName(m[2]),
+      },
+    ],
+  },
+];
+
+// ---------------------------------------------------------------------------
+// Scanning
+// ---------------------------------------------------------------------------
+
+const lineBreakAfter = (text, from) => {
+  const at = text.indexOf('\n', from);
+  return at === -1 ? text.length : at;
+};
+
+/** The fenced code blocks of a Markdown text: [{ start, end, lang }] where start..end is the body (a fence left open runs to the end). */
+function findFences(text) {
+  const blocks = [];
+  let open = null;
+  for (const line of text.matchAll(/^[ \t]{0,3}(`{3,20}|~{3,20})[ \t]{0,8}([^\s`]{0,40})/gm)) {
+    if (open === null) open = { start: lineBreakAfter(text, line.index + line[0].length), char: line[1][0], size: line[1].length, lang: line[2].toLowerCase() };
+    else if (line[1][0] === open.char && line[1].length >= open.size && line[2] === '') {
+      blocks.push({ start: open.start, end: line.index, lang: open.lang });
+      open = null;
+    }
+  }
+  if (open !== null) blocks.push({ start: open.start, end: text.length, lang: open.lang });
+  return blocks;
+}
+
+function buildNewlineIndex(text) {
+  const positions = [];
+  let i = -1;
+  while ((i = text.indexOf('\n', i + 1)) !== -1) positions.push(i);
+  return positions;
+}
+
+/** 1-based line number of the character at `index`. */
+function lineAt(newlines, index) {
+  let lo = 0;
+  let hi = newlines.length;
+  while (lo < hi) {
+    const mid = (lo + hi) >> 1;
+    if (newlines[mid] < index) lo = mid + 1;
+    else hi = mid;
+  }
+  return lo + 1;
+}
+
+/** True when the allow marker sits on the given 1-based line. Cached per line: one minified line can hold thousands of matches. */
+function lineHasAllowMarker(text, newlines, line, cache) {
+  let known = cache.get(line);
+  if (known === undefined) {
+    const start = line === 1 ? 0 : newlines[line - 2] + 1;
+    const end = line - 1 < newlines.length ? newlines[line - 1] : text.length;
+    known = text.slice(start, end).includes(ALLOW_MARKER);
+    cache.set(line, known);
+  }
+  return known;
+}
+
+/**
+ * Scan one file's text. Pure: no I/O.
+ * @returns {{path: string, line: number, rule: string}[]} findings, never the matched text
+ */
+export function scanText(filePath, text) {
+  return scanRanges(filePath, text).map(({ path: p, line, rule }) => ({ path: p, line, rule }));
+}
+
+/**
+ * What each finding matched, kept apart from the finding itself so that no report, spread copy or serialisation of a finding can carry it.
+ * Held in memory for the run, compared with `evidenceKey`, and never printed, logged or stored.
+ */
+const EVIDENCE = new WeakMap();
+/** Identity of a finding inside one run: its rule and the two evidence strings, with a length prefix so they cannot run together. */
+const evidenceKey = (finding) => {
+  const [whole, value] = EVIDENCE.get(finding) ?? ['', ''];
+  return `${finding.rule}\t${whole.length}\t${whole}${value}`;
+};
+
+/** Like scanText, but each finding also carries `lastLine`, the last line its match spans, and valueFirst..valueLast, the line(s) of a separate value field. */
+function scanRanges(filePath, text) {
+  const findings = [];
+  const seen = new Set();
+  const markerCache = new Map();
+  const lockfile = isLockfile(filePath);
+  if (lockfile) text = sanitizeLockfile(text); // ordinary integrity digests are not scanned
+  const formats = credentialFormats(filePath);
+  const strict = STRICT_CREDENTIAL_FORMATS.some((tag) => formats.has(tag));
+  let newlines = null;
+  const newlineIndex = () => (newlines ??= buildNewlineIndex(text));
+  const occurrences = new Map();
+  let fences = null;
+  const ctx = {
+    path: filePath.split(path.sep).join('/'),
+    mode: fileMode(filePath),
+    formats,
+    strict,
+    lockfile,
+    xml: isXmlConfigPath(filePath),
+    minStrong: strict ? CREDENTIAL_FILE_MIN_LENGTH : MIN_STRONG_CONFIG_LENGTH,
+    runsToEndOfLine: valueRunsToEndOfLine(filePath),
+    catalog: isMessageCatalogPath(filePath),
+    properties: isPropertiesPath(filePath),
+    // Start of the line holding `index` and the index of its line break (text.length when it has none): O(log n).
+    lineStart(index) {
+      const line = lineAt(newlineIndex(), index);
+      return line === 1 ? 0 : newlineIndex()[line - 2] + 1;
+    },
+    // Is there an occurrence of `needle` within `distance` characters before `index`? Occurrences are listed once per file.
+    hasBefore(needle, index, distance) {
+      let list = occurrences.get(needle);
+      if (list === undefined) {
+        list = [];
+        for (let at = text.indexOf(needle); at !== -1 && list.length < 200_000; at = text.indexOf(needle, at + needle.length)) list.push(at);
+        occurrences.set(needle, list);
+      }
+      let lo = 0;
+      let hi = list.length;
+      while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (list[mid] <= index) lo = mid + 1;
+        else hi = mid;
+      }
+      if (lo === 0) return false;
+      // A list that hit its cap cannot say what lies beyond it: fail closed.
+      return index - list[lo - 1] <= distance || (list.length >= 200_000 && lo === list.length);
+    },
+    lineEnd(index) {
+      const line = lineAt(newlineIndex(), index);
+      return line - 1 < newlineIndex().length ? newlineIndex()[line - 1] : text.length;
+    },
+    // The language tag of the Markdown code fence that holds `index` ('' for a bare fence), or undefined outside every fence.
+    fenceLang(index) {
+      fences ??= findFences(text);
+      let lo = 0;
+      let hi = fences.length;
+      while (lo < hi) {
+        const mid = (lo + hi) >> 1;
+        if (fences[mid].end <= index) lo = mid + 1;
+        else hi = mid;
+      }
+      return lo < fences.length && fences[lo].start <= index ? fences[lo].lang : undefined;
+    },
+  };
+  for (const rule of RULES) {
+    if (lockfile && !rule.lockfile) continue; // lockfiles: targeted rules only (see LOCKFILE_NAMES)
+    if (rule.appliesTo && !rule.appliesTo(ctx)) continue;
+    // The name hint is a speed-up for ordinary files; a credential file is always scanned in full.
+    if (rule.hint && !strict && !rule.hint.test(text)) continue;
+    for (const matcher of rule.matchers) {
+      if (matcher.appliesTo && !matcher.appliesTo(ctx)) continue;
+      if (matcher.hint && !matcher.hint.test(text)) continue;
+      for (const match of text.matchAll(matcher.pattern)) {
+        newlineIndex();
+        const line = lineAt(newlines, match.index);
+        const key = `${rule.id}:${line}`;
+        if (seen.has(key)) continue; // already reported for this line: nothing to decide, and it keeps a hostile line cheap
+        if (matcher.accept && !matcher.accept(match, ctx)) continue;
+        if (isPathSample(ctx.path, text.slice(match.index, match.spanEnd ?? match.index + match[0].length))) continue;
+        const lastLine = lineAt(newlines, match.index + Math.max((match.spanEnd ?? match.index + match[0].length) - match.index - 1, 0));
+        // The lines the finding rests on (--history and --range blame a commit that added one of them): the match, plus the
+        // separate value of a name/value pair (valueFirst..valueLast; the match itself when there is none).
+        const valueFirst = match.attrStart === undefined ? line : lineAt(newlines, match.attrStart);
+        const valueLast = match.attrEnd === undefined ? lastLine : lineAt(newlines, Math.max(match.attrEnd - 1, 0));
+        // The marker may sit on any line the match spans (a match can run across lines), or on a line of the separate value
+        // of a name/value pair, in either order. A marker on an unrelated line between the two does not count.
+        let allowed = false;
+        for (let l = line; l <= lastLine && !allowed; l += 1) allowed = lineHasAllowMarker(text, newlines, l, markerCache);
+        if (!allowed && match.attrStart !== undefined && !match.attrCoarse) {
+          for (let l = valueFirst; l <= valueLast && !allowed; l += 1) allowed = lineHasAllowMarker(text, newlines, l, markerCache);
+        }
+        if (allowed) continue;
+        seen.add(key);
+        // EVIDENCE lets --history and --range tell whether a merge result carries a match that one of its parents already had. It
+        // is compared in memory during the run only (a Set of strings): no hash of it is made, and it is never printed, logged or stored.
+        const matchEnd = match.spanEnd ?? match.index + match[0].length;
+        const evidence = [text.slice(match.index, matchEnd), match.attrStart === undefined || match.attrCoarse ? '' : text.slice(match.attrStart, match.attrEnd ?? matchEnd)];
+        const finding = { path: filePath, line, lastLine, valueFirst, valueLast, rule: rule.id };
+        EVIDENCE.set(finding, evidence);
+        findings.push(finding);
+      }
+    }
+  }
+  return findings.sort((a, b) => a.line - b.line || a.rule.localeCompare(b.rule));
+}
+
+function git(args, cwd) {
+  const result = spawnSync('git', args, { cwd, maxBuffer: 512 * 1024 * 1024 });
+  if (result.error || result.status !== 0) {
+    throw new Error(`git ${args[0]} failed (is this a git repository?)`);
+  }
+  return result.stdout;
+}
+
+function findRepoRoot(cwd) {
+  return git(['rev-parse', '--show-toplevel'], cwd).toString('utf8').trim();
+}
+
+function readHead(absolutePath) {
+  const fd = openSync(absolutePath, 'r');
+  try {
+    const buffer = Buffer.alloc(SNIFF_BYTES);
+    const bytes = readSync(fd, buffer, 0, SNIFF_BYTES, 0);
+    return buffer.subarray(0, bytes);
+  } finally {
+    closeSync(fd);
+  }
+}
+
+const strictUtf8 = new TextDecoder('utf-8', { fatal: true });
+
+/** A path for messages and for the file-mode rules. Valid UTF-8 as is; otherwise every byte above 0x7e is shown as \xNN. */
+function displayPath(raw) {
+  try {
+    return strictUtf8.decode(raw);
+  } catch {
+    return raw.toString('latin1').replace(/[\u007f-ÿ]/g, (c) => `\\x${c.charCodeAt(0).toString(16).padStart(2, '0')}`);
+  }
+}
+
+/**
+ * Every tracked path from `git ls-files -s -z`, as raw bytes. The list is NUL-delimited and never decoded as a
+ * whole: a file name may hold bytes that are not UTF-8 (allowed on Unix), and a lossy decode would name a file
+ * that does not exist. Unmerged paths appear once per stage: they are listed once, with every distinct object and the
+ * mode it has in its own stage (the stages of one path can differ: a gitlink in one, a regular file in another).
+ * @returns {{raw: Buffer, objects: {mode: string, sha: string}[]}[]} every distinct (mode, object id) of the path
+ */
+function listTracked(root) {
+  const out = git(['ls-files', '-s', '-z'], root);
+  const entries = new Map();
+  let start = 0;
+  while (start < out.length) {
+    let end = out.indexOf(0, start);
+    if (end === -1) end = out.length;
+    const record = out.subarray(start, end);
+    start = end + 1;
+    if (record.length === 0) continue;
+    const tab = record.indexOf(9);
+    const [mode, sha] = tab === -1 ? [] : record.subarray(0, tab).toString('latin1').split(' ');
+    if (!mode || !sha) throw new Error('git ls-files printed a record this scanner cannot parse');
+    const raw = Buffer.from(record.subarray(tab + 1));
+    const key = raw.toString('latin1');
+    const known = entries.get(key);
+    if (!known) entries.set(key, { raw, objects: [{ mode, sha }] });
+    else if (!known.objects.some((o) => o.mode === mode && o.sha === sha)) known.objects.push({ mode, sha });
+  }
+  return [...entries.values()];
+}
+
+/**
+ * Git object ids (as `git hash-object --no-filters` computes them, in the repository's own hash) of working-tree files, so the
+ * scanner never hashes file content itself. One git process covers every file whose path can be sent on a line; a file
+ * that cannot (newline, carriage return or a leading quote in its name) is hashed in its own process, streamed from an open
+ * descriptor so a file over the size limit is never held in memory. A file git cannot hash is missing from the map, and
+ * the caller then treats the index versions as differing, which is the safe direction.
+ * @param {Buffer[]} absolutePaths
+ * @returns {Map} the path bytes as a latin1 string -> object id
+ */
+function hashWorkingFiles(root, absolutePaths) {
+  const ids = new Map();
+  const batch = [];
+  const single = [];
+  for (const absolute of absolutePaths) {
+    if (absolute.includes(10) || absolute.includes(13) || absolute[0] === 34) single.push(absolute);
+    else batch.push(absolute);
+  }
+  if (batch.length > 0) {
+    const input = Buffer.concat(batch.flatMap((absolute) => [absolute, Buffer.from('\n')]));
+    const run = spawnSync('git', ['hash-object', '--no-filters', '--stdin-paths'], { cwd: root, input, maxBuffer: 64 * 1024 * 1024 });
+    const lines = run.error || run.status !== 0 ? [] : run.stdout.toString('latin1').split('\n');
+    if (lines.length === batch.length + 1) batch.forEach((absolute, i) => ids.set(absolute.toString('latin1'), lines[i]));
+    else single.push(...batch); // any doubt about the batch: hash each file on its own
+  }
+  for (const absolute of single) {
+    let fd;
+    try {
+      fd = openSync(absolute, 'r');
+      const run = spawnSync('git', ['hash-object', '--no-filters', '--stdin'], { cwd: root, stdio: [fd, 'pipe', 'pipe'] });
+      const id = run.error || run.status !== 0 ? '' : run.stdout.toString('latin1').trim();
+      if (id !== '') ids.set(absolute.toString('latin1'), id);
+    } catch {
+      // unreadable: left out of the map
+    } finally {
+      if (fd !== undefined) closeSync(fd);
+    }
+  }
+  return ids;
+}
+
+/** The object id git gives to `bytes` stored as a blob ('' when git cannot say). */
+function hashBytes(root, bytes) {
+  const run = spawnSync('git', ['hash-object', '--no-filters', '--stdin'], { cwd: root, input: bytes, maxBuffer: 1024 * 1024 });
+  return run.error || run.status !== 0 ? '' : run.stdout.toString('latin1').trim();
+}
+
+/**
+ * The content of several blobs from the index, in two `git cat-file --batch*` calls in total: sizes first (so a blob
+ * over the size limit is never read into memory), then the contents.
+ * @returns {Map} bytes is null for a blob over MAX_LOCKFILE_BYTES (the callers apply the per-path limit); a blob git
+ *   could not produce is missing from the map (the caller treats that as unreadable)
+ */
+function readIndexBlobs(root, shas) {
+  const result = new Map();
+  if (shas.length === 0) return result;
+  const request = (mode, ids) => {
+    const run = spawnSync('git', ['cat-file', mode], { cwd: root, input: `${ids.join('\n')}\n`, maxBuffer: 1024 * 1024 * 1024 });
+    if (run.error || run.status !== 0) throw new Error(`git cat-file failed`);
+    return run.stdout;
+  };
+  const sizes = new Map();
+  for (const line of request('--batch-check', shas).toString('latin1').split('\n')) {
+    const parts = line.split(' ');
+    if (parts.length === 3 && parts[1] === 'blob' && /^\d+$/.test(parts[2])) sizes.set(parts[0], Number(parts[2]));
+  }
+  const readable = shas.filter((sha) => sizes.has(sha) && sizes.get(sha) <= MAX_LOCKFILE_BYTES);
+  for (const sha of shas) {
+    if (sizes.has(sha) && sizes.get(sha) > MAX_LOCKFILE_BYTES) result.set(sha, { size: sizes.get(sha), bytes: null });
+  }
+  if (readable.length === 0) return result;
+  const stdout = request('--batch', readable);
+  let cursor = 0;
+  for (const sha of readable) {
+    const headerEnd = stdout.indexOf(10, cursor);
+    if (headerEnd === -1) break;
+    const [id, type, size] = stdout.subarray(cursor, headerEnd).toString('latin1').split(' ');
+    if (id !== sha || type !== 'blob' || !/^\d+$/.test(size ?? '')) break;
+    const bodyStart = headerEnd + 1;
+    const bodyEnd = bodyStart + Number(size);
+    if (bodyEnd > stdout.length) break;
+    result.set(sha, { size: Number(size), bytes: Buffer.from(stdout.subarray(bodyStart, bodyEnd)) });
+    cursor = bodyEnd + 1;
+  }
+  return result;
+}
+
+/**
+ * Scan every git-tracked file under `root`: the working-tree content (what CI checked out) AND the version staged in
+ * the index whenever the two differ, so a secret cannot be staged and then swapped for a placeholder in the working
+ * tree before the commit. The two are compared by git blob id (one hash of bytes already read); only files that
+ * differ cost an extra read, batched into one `git cat-file --batch`. A CI checkout has none.
+ *  - gitlinks (submodule commit pointers: no content here) and files with binary content are skipped on purpose,
+ *    and counted in `skipped`; lockfiles are scanned with the lockfile rules (see isLockfile);
+ *  - a symlink is scanned as its link target;
+ *  - a file the index lists but the working tree no longer has (deleted, or skip-worktree) is scanned from the
+ *    index blob instead, so nothing tracked goes unexamined (`fromIndex` names them);
+ *  - an unmerged path is scanned with every stage's blob, and the working-tree file besides;
+ *  - a file that exists but cannot be read (permissions, wrong type, I/O error) is NOT scanned and is listed in
+ *    `unreadable`; a text file over the size limit (either version) is listed in `oversize`. The CLI fails on both.
+ * A finding in a file that has a differing second version carries `source`: 'index' or 'working tree'.
+ * @returns {{findings: object[], scanned: number, skipped: Record, oversize: string[], unreadable: string[], fromIndex: string[], differing: string[]}}
+ */
+export function scanTree(root) {
+  const findings = [];
+  const skipped = {};
+  const oversize = [];
+  const unreadable = [];
+  const fromIndex = [];
+  const differing = [];
+  let scanned = 0;
+  const skip = (reason) => {
+    skipped[reason] = (skipped[reason] ?? 0) + 1;
+  };
+  const rootBytes = Buffer.from(root);
+  const pending = []; // index versions still to read: { file, sha, missing }
+  const tracked = listTracked(root);
+  const workingIds = hashWorkingFiles(root, tracked.flatMap(({ raw }) => {
+    const absolute = Buffer.concat([rootBytes, Buffer.from(path.sep), raw]);
+    try {
+      return lstatSync(absolute).isFile() ? [absolute] : [];
+    } catch {
+      return [];
+    }
+  }));
+  for (const { raw, objects } of tracked) {
+    const file = displayPath(raw);
+    // Only a gitlink (a submodule commit pointer, no content here) is skipped, and only its own stage: every other
+    // stage of an unmerged path is a file and is scanned, whatever mode the gitlink stage has.
+    const files = objects.filter((o) => o.mode !== '160000');
+    if (files.length < objects.length) skip('submodule');
+    if (files.length === 0) continue;
+    const shas = [...new Set(files.map((o) => o.sha))];
+    const hasLink = files.some((o) => o.mode === '120000');
+    const hasFile = files.some((o) => o.mode !== '120000');
+    const absolute = Buffer.concat([rootBytes, Buffer.from(path.sep), raw]);
+    let bytes;
+    let missing = false;
+    try {
+      const stat = lstatSync(absolute);
+      if (stat.isSymbolicLink()) {
+        if (!hasLink) throw new Error('not a regular file');
+        bytes = readlinkSync(absolute, 'buffer');
+      } else {
+        if (!stat.isFile() || !hasFile) throw new Error(hasFile ? 'not a regular file' : 'not a symlink');
+        if (stat.size > sizeLimit(file)) {
+          // Too large to read into memory here, but the staged version is still checked first: the working-tree copy
+          // (a big binary, or an oversize text file) must not hide a differing index blob. The copy is hashed in chunks.
+          const workingId = workingIds.get(absolute.toString('latin1')) ?? '';
+          const otherVersions = shas.filter((blob) => blob !== workingId);
+          if (otherVersions.length > 0) {
+            differing.push(file);
+            for (const blob of otherVersions) pending.push({ file, sha: blob, missing: false });
+          }
+          if (isBinaryContent(readHead(absolute))) skip('binary');
+          else oversize.push(file);
+          continue;
+        }
+        bytes = readFileSync(absolute);
+      }
+    } catch (error) {
+      if (error?.code !== 'ENOENT') {
+        unreadable.push(file);
+        // The working-tree entry cannot be read, but the staged versions still can: they are scanned too.
+        for (const blob of shas) pending.push({ file, sha: blob, missing: false });
+        continue;
+      }
+      missing = true;
+    }
+    if (missing) {
+      fromIndex.push(file);
+      for (const blob of shas) pending.push({ file, sha: blob, missing: true });
+      continue;
+    }
+    // The index blob that differs from the working-tree bytes (all of them for an unmerged path) is scanned too.
+    const workingId = workingIds.get(absolute.toString('latin1')) ?? hashBytes(root, bytes);
+    const otherVersions = shas.filter((blob) => blob !== workingId);
+    if (otherVersions.length > 0) {
+      differing.push(file);
+      for (const blob of otherVersions) pending.push({ file, sha: blob, missing: false });
+    }
+    const text = decodeText(bytes);
+    if (text === null) {
+      skip('binary');
+      continue;
+    }
+    scanned += 1;
+    for (const finding of scanText(file, text)) {
+      findings.push(otherVersions.length > 0 ? { ...finding, source: 'working tree' } : finding);
+    }
+  }
+
+  if (pending.length > 0) {
+    let blobs = new Map();
+    try {
+      blobs = readIndexBlobs(root, [...new Set(pending.map((p) => p.sha))]);
+    } catch {
+      // every pending version stays unreadable below
+    }
+    const reported = new Set();
+    for (const { file, sha, missing } of pending) {
+      const blob = blobs.get(sha);
+      if (blob === undefined) {
+        if (!reported.has(`${file}\0u`)) unreadable.push(file);
+        reported.add(`${file}\0u`);
+        continue;
+      }
+      if (blob.bytes === null || blob.size > sizeLimit(file)) {
+        if (!reported.has(`${file}\0o`) && !oversize.includes(file)) oversize.push(file);
+        reported.add(`${file}\0o`);
+        continue;
+      }
+      const text = decodeText(blob.bytes);
+      if (text === null) {
+        skip(missing ? 'binary' : 'binary (staged version)');
+        continue;
+      }
+      if (missing) scanned += 1;
+      for (const finding of scanText(file, text)) findings.push({ ...finding, source: 'index' });
+    }
+  }
+  return { findings, scanned, skipped, oversize, unreadable, fromIndex, differing };
+}
+
+/** Decode a path git printed C-style quoted ("b/we\"ird/a.env", octal escapes for control bytes). */
+function unquoteGitPath(quoted) {
+  const escapes = { a: 7, b: 8, f: 12, n: 10, r: 13, t: 9, v: 11, '\\': 92, '"': 34 };
+  const bytes = [];
+  const body = quoted.slice(1);
+  for (let i = 0; i < body.length; i += 1) {
+    const ch = body[i];
+    if (ch === '"') break;
+    if (ch !== '\\') {
+      bytes.push(...Buffer.from(ch, 'utf8'));
+    } else if (/[0-7]/.test(body[i + 1] ?? '')) {
+      const octal = /^[0-7]{1,3}/.exec(body.slice(i + 1))[0];
+      bytes.push(parseInt(octal, 8) & 0xff);
+      i += octal.length;
+    } else {
+      bytes.push(escapes[body[i + 1]] ?? body[i + 1].charCodeAt(0));
+      i += 1;
+    }
+  }
+  return Buffer.from(bytes).toString('utf8');
+}
+
+const printable = (p) => String(p).replace(/[\u0000-\u001f\u007f]/g, '?');
+
+/** First line of git's stderr, cleaned. In the failure modes seen so far it names refs and objects, never file content. */
+function summarizeStderr(stderr) {
+  const first = stderr.trim().split('\n')[0] ?? '';
+  return first === '' ? '' : `: ${printable(first).slice(0, 200)}`;
+}
+
+// Unchanged context kept around each change, so a split "name: X / value: Y" pair whose name line did not change can
+// still be recognised. Context lines are never reported by themselves. It is derived from the constants the pair matcher
+// (and the other multi-line rules: PEM headers, netrc tokens, mapping keys) look back and forward with, so the two cannot
+// drift apart. The YAML window and the multi-line literal reader count LINES; the brace and XML windows count CHARACTERS.
+// A character window is kept as characters: context lines are retained until their text (each line plus its newline, as
+// the scanned text holds it) covers the window, however short or blank the lines are. No line length is assumed anywhere.
+const HISTORY_CONTEXT_LINES = Math.max(
+  PAIR_YAML_LINES,
+  MULTILINE_MAX_LINES + 1, // a heredoc / triple-quoted body and its closing line
+);
+const HISTORY_CONTEXT_CHARS = Math.max(PAIR_BACK_CHARS, PAIR_FORWARD_CHARS, PAIR_XML_CHARS);
+// The unified context git is asked for. Every line is at least its own newline, so a window of N characters never spans
+// more than N lines: asking git for that many lines (it stops at the file's ends) always supplies enough to fill the window.
+export const HISTORY_CONTEXT = Math.max(HISTORY_CONTEXT_LINES, HISTORY_CONTEXT_CHARS);
+// A lockfile is scanned with single-line rules only (no name/value pairing) and one multi-line rule, the PEM private key block
+// (private-key-block: its header, optional header fields and the key material are separate lines). So a bump that touches a big
+// lockfile in many places must not drag hundreds of unchanged lines around every change into the text that is size-checked and
+// scanned: it keeps HISTORY_LOCKFILE_CONTEXT, and the wider text window only where a `-----BEGIN` line makes a PEM block possible
+// (see pemBoundary), which costs nothing for a lockfile without key material.
+const HISTORY_LOCKFILE_CONTEXT = { lines: 2, chars: 0 };
+const HISTORY_TEXT_CONTEXT = { lines: HISTORY_CONTEXT_LINES, chars: HISTORY_CONTEXT_CHARS };
+const PEM_BOUNDARY = '-----BEGIN';
+/** The unchanged lines that stay in front of an added line of a lockfile: the last few, or all from the nearest `-----BEGIN` line on. */
+function lockfileLookbehind(recent) {
+  for (let i = recent.length - 1; i >= 0; i -= 1) {
+    if (recent[i].includes(PEM_BOUNDARY)) return { held: recent.slice(Math.min(i, Math.max(recent.length - HISTORY_LOCKFILE_CONTEXT.lines, 0))), pem: true };
+  }
+  return { held: recent.slice(-HISTORY_LOCKFILE_CONTEXT.lines), pem: false };
+}
+
+/**
+ * The first SNIFF_BYTES bytes of `path` as it is in `commit` (`git cat-file blob :`), or null when git
+ * cannot produce them. The read is bounded: spawnSync stops the child at SNIFF_BYTES, so a huge blob is never held.
+ */
+function blobHead(root, commit, filePath) {
+  const run = spawnSync('git', ['cat-file', 'blob', filePath === null ? commit : `${commit}:${filePath}`], {
+    cwd: root,
+    maxBuffer: SNIFF_BYTES,
+    stdio: ['ignore', 'pipe', 'ignore'],
+  });
+  const complete = run.status === 0 && !run.error;
+  const cut = run.error?.code === 'ENOBUFS' && run.stdout && run.stdout.length > 0; // longer than SNIFF_BYTES: the head is what was wanted
+  return complete || cut ? Buffer.from(run.stdout).subarray(0, SNIFF_BYTES) : null;
+}
+
+/** True only when the tree scan would also skip this version as binary: the same isBinaryContent test on the same first 8 KB. */
+const versionIsVerifiedBinary = (root, commit, filePath) => {
+  const head = blobHead(root, commit, filePath);
+  return head !== null && isBinaryContent(head);
+};
+
+/** The path a `diff --git a/P b/P` (or `diff --cc P`) header names, or null when it cannot be told exactly (quoted, renamed). */
+function diffHeaderPath(header) {
+  const combined = /^diff --(?:cc|combined) (.+)$/.exec(header);
+  if (combined) return combined[1].startsWith('"') ? unquoteGitPath(combined[1]) : combined[1];
+  const rest = header.slice('diff --git '.length);
+  const half = (rest.length - 5) / 2;
+  if (!Number.isInteger(half) || half < 1 || rest.startsWith('"')) return null;
+  const left = rest.slice(2, 2 + half);
+  return rest.startsWith('a/') && rest.slice(2 + half, 5 + half) === ' b/' && rest.slice(5 + half) === left ? left : null;
+}
+
+// Longest git output line that is held whole. A longer line is fed to the parser cut at this length and the rest is
+// dropped: no line that long can be a header, and an added line that long is over every size limit (it is reported
+// as oversize or, if the version is a verified binary, skipped), so nothing that is scanned as text is lost.
+const MAX_HELD_LINE_CHARS = MAX_LOCKFILE_BYTES + 2;
+
+// ---------------------------------------------------------------------------
+// Merge results
+// ---------------------------------------------------------------------------
+
+/** The object ids of a `parents   ...` line (none for a root commit, which prints a trailing blank), or null for any other text. */
+function parseParentsLine(line) {
+  if (!line.startsWith('parents')) return null;
+  const rest = line.slice('parents'.length);
+  if (rest === '' || rest === ' ') return [];
+  if (rest[0] !== ' ') return null;
+  const ids = rest.slice(1).split(' ');
+  if (ids[ids.length - 1] === '') ids.pop();
+  return ids.every((id) => /^[0-9a-f]{40,64}$/.test(id)) ? ids : null;
+}
+
+const MERGE_BATCH = 25; // merge commits per `git diff-tree --stdin` call, so one call's output stays small
+const ZERO_OBJECT = /^0+$/;
+
+/**
+ * `git diff-tree --stdin -m -r -z --no-renames --raw` output as { commit id -> one Map per parent, in parent order } where each Map
+ * holds path (latin1 of its raw bytes) -> { raw, oldMode, newMode, oldOid, newOid, status } of the merge result against that parent.
+ * A parent whose tree equals the merge result has no entries (git may print no group for it: the callers count the groups).
+ */
+function parseMergeDiffs(out) {
+  const groups = new Map();
+  let current = null;
+  let start = 0;
+  const next = () => {
+    if (start >= out.length) return null;
+    let end = out.indexOf(0, start);
+    if (end === -1) end = out.length;
+    const token = out.subarray(start, end);
+    start = end + 1;
+    return token;
+  };
+  for (let token = next(); token !== null; token = next()) {
+    if (token.length === 0) continue;
+    if (token[0] === 0x3a) {
+      const meta = token.toString('latin1').slice(1).split(' ');
+      const raw = next();
+      if (meta.length !== 5 || raw === null || current === null) throw new Error('git diff-tree printed output this scanner cannot parse');
+      current.set(raw.toString('latin1'), { raw: Buffer.from(raw), oldMode: meta[0], newMode: meta[1], oldOid: meta[2], newOid: meta[3], status: meta[4] });
+    } else {
+      const id = token.toString('latin1');
+      if (!/^[0-9a-f]{40,64}$/.test(id)) throw new Error('git diff-tree printed output this scanner cannot parse');
+      current = new Map();
+      if (!groups.has(id)) groups.set(id, []);
+      groups.get(id).push(current);
+    }
+  }
+  return groups;
+}
+
+/** The first SNIFF_BYTES bytes of a blob given by object id, or null when git cannot produce them (see blobHead). */
+function objectHead(root, oid) {
+  return blobHead(root, oid, null);
+}
+
+/**
+ * A merge can complete a credential that neither parent held: one parent adds `"name": "JWT_SECRET"`, the other adds a nearby
+ * `"value": "..."`, or one adds a PEM header and the other its key material, and git merges both without a conflict. `git log --cc`
+ * shows only lines that differ from EVERY parent, so it shows none of it. This looks at the merge RESULT instead: for each file
+ * where the result differs from every parent (a file equal to one parent holds nothing that parent did not), it scans the result
+ * and reports a finding only when no parent's version of that file has the same finding (same rule, same matched text, compared
+ * by digest and never printed). A credential a parent already held is that parent's, reported wherever the parent's side is
+ * scanned; blaming it on the merge would blame a branch for commits that come in from its base.
+ * @returns {{hits: Map, oversize: number, oversizeLimits: Map, unscanned: number}}
+ */
+function scanMergeResults(root, merges) {
+  const hits = new Map();
+  const oversizeLimits = new Map();
+  let oversize = 0;
+  let unscanned = 0;
+  for (let from = 0; from < merges.length; from += MERGE_BATCH) {
+    const batch = merges.slice(from, from + MERGE_BATCH);
+    const run = spawnSync('git', ['diff-tree', '--stdin', '-m', '-r', '-z', '--no-renames', '--raw'], {
+      cwd: root,
+      input: `${batch.map((m) => m.commit).join('\n')}\n`,
+      maxBuffer: 1024 * 1024 * 1024,
+    });
+    if (run.error || run.status !== 0) throw new Error('git diff-tree failed while reading merge commits');
+    const groups = parseMergeDiffs(run.stdout);
+    const candidates = [];
+    for (const { commit, parents } of batch) {
+      const perParent = groups.get(commit) ?? [];
+      if (perParent.length > parents.length) throw new Error('git diff-tree printed more parents than a merge has');
+      if (perParent.length < parents.length) continue; // the result equals a parent's tree: nothing of it is new
+      for (const [key, entry] of perParent[0]) {
+        if (entry.status === 'D' || entry.newMode === '160000') continue;
+        const others = perParent.map((group) => group.get(key));
+        if (others.some((other) => other === undefined || other.status === 'D')) continue; // equal to a parent's version
+        candidates.push({
+          commit,
+          file: displayPath(entry.raw),
+          oid: entry.newOid,
+          parentOids: others.map((other) => (other.oldMode === '160000' || ZERO_OBJECT.test(other.oldOid) ? null : other.oldOid)),
+        });
+      }
+    }
+    if (candidates.length === 0) continue;
+    const blobs = readIndexBlobs(root, [...new Set(candidates.map((c) => c.oid))]);
+    for (const candidate of candidates) {
+      const { commit, file } = candidate;
+      const blob = blobs.get(candidate.oid);
+      if (blob === undefined) {
+        unscanned += 1;
+        continue;
+      }
+      if (blob.bytes === null || blob.size > sizeLimit(file)) {
+        const head = objectHead(root, candidate.oid);
+        if (head !== null && isBinaryContent(head)) continue;
+        oversize += 1;
+        unscanned += 1;
+        oversizeLimits.set(sizeLimit(file), (oversizeLimits.get(sizeLimit(file)) ?? 0) + 1);
+        continue;
+      }
+      const text = decodeText(blob.bytes);
+      if (text === null) continue;
+      const found = scanRanges(file, text);
+      if (found.length === 0) continue;
+      // What the parents already had. A parent version that cannot be read or scanned contributes nothing, which reports more.
+      const known = new Set();
+      const parentBlobs = readIndexBlobs(root, [...new Set(candidate.parentOids.filter((oid) => oid !== null))]);
+      for (const oid of candidate.parentOids) {
+        const parentBlob = oid === null ? undefined : parentBlobs.get(oid);
+        if (parentBlob === undefined || parentBlob.bytes === null || parentBlob.size > sizeLimit(file)) continue;
+        const parentText = decodeText(parentBlob.bytes);
+        if (parentText === null) continue;
+        for (const finding of scanRanges(file, parentText)) known.add(evidenceKey(finding));
+      }
+      for (const finding of found) {
+        if (known.has(evidenceKey(finding))) continue;
+        const key = `${commit}\t${file}\t${finding.rule}`;
+        const entry = hits.get(key) ?? { commit, path: file, rule: finding.rule, count: 0 };
+        entry.count += 1;
+        hits.set(key, entry);
+      }
+    }
+  }
+  return { hits, oversize, oversizeLimits, unscanned };
+}
+
+/**
+ * Scan every commit reachable from any ref, reporting only matches that touch a line the commit ADDED.
+ * Unchanged context lines are scanned together with the added ones (so multi-line rules can see the
+ * name next to a new value) but a match made only of context lines belongs to an earlier commit.
+ * Merge commits are shown as combined diffs (--cc), so only lines that the merge itself introduced
+ * (conflict resolutions) are scanned; everything else was added by a parent and is reported there.
+ * @returns {Promise<{hits: {commit: string, path: string, rule: string, count: number}[], commits: number, oversize: number, oversizeLimits: Map, unscanned: number, skipped: Record}>}
+ * `unscanned` counts every file version whose added lines were not examined (oversize, or content git would not show).
+ * `skipped` counts the file versions left out on purpose, as the tree scan does: a verified binary (see isBinaryContent, checked
+ * on the version's own first 8 KB, so a binary asset over the size limit does not count as oversize) is `binary`.
+ * `revisions` selects the commits (default every ref; range mode passes ` --not `, so the walk and the diffs
+ * cost what the range holds, not what the repository holds); `maxCount` limits the walk (range mode with an unknown base).
+ */
+async function scanHistory(root, { revisions = ['--all'], maxCount = null } = {}) {
+  const child = spawn(
+    'git',
+    [
+      '-c', 'core.quotepath=false',
+      'log', ...(maxCount === null ? [] : [`--max-count=${maxCount}`]), '--no-color', '--no-ext-diff', '--no-textconv', '--no-renames', '--text',
+      '-p', '--cc', `-U${HISTORY_CONTEXT}`, '--format=commit %H%nparents %P',
+      ...revisions, '--',
+    ],
+    { cwd: root, stdio: ['ignore', 'pipe', 'pipe'] },
+  );
+  let stderr = '';
+  child.stderr.on('data', (chunk) => {
+    if (stderr.length < 4096) stderr += chunk.toString('utf8');
+  });
+  const exited = new Promise((resolve, reject) => {
+    child.on('error', reject);
+    child.on('close', resolve);
+  });
+
+  const hits = new Map();
+  let commits = 0;
+  let oversize = 0;
+  const oversizeLimits = new Map(); // size limit in bytes -> file versions over it (the limit differs per path)
+  let unscanned = 0; // file versions whose added lines were NOT examined, for any reason (oversize included)
+  const skipped = {}; // file versions left out on purpose, by reason: the same accounting as the tree scan ('binary')
+  let commit = null;
+  let file = null;
+  let headerPath = null; // the path named by the current `diff` header, for versions git prints as one "Binary files" line
+  let inHunk = false;
+  let parents = 1;
+  let lines = []; // added lines and the context around them (see contextFor) of the current file; gaps are a blank line
+  let addedLines = new Set(); // 1-based indexes into `lines` of the lines this commit added
+  let addedAny = false; // this version had an added line, even one that overflowed the size limit before it could be indexed
+  let textChars = 0; // characters held in `lines`
+  let overflow = false; // the retained text passed the file's size limit: stop collecting, report it as oversize
+  let recent = []; // context lines seen since the last kept line: the newest ones that cover contextFor(file) (lines AND characters)
+  let recentChars = 0; // characters (lines plus newlines) in `recent`
+  let dropped = false; // context lines were left out since the last kept line
+  let afterLines = 0; // context lines still to keep after the last added line ...
+  let afterChars = 0; // ... and characters still to cover; a line is kept while either is left
+  // The context kept after an added line. A lockfile keeps little, unless the line is (or follows) a PEM boundary.
+  const contextFor = (path, pem = false) => (isLockfile(path) && !pem ? HISTORY_LOCKFILE_CONTEXT : HISTORY_TEXT_CONTEXT);
+  // Drop the oldest held context line while the rest still covers the window (enough lines AND enough characters), or
+  // while what is held could not fit the file's size limit anyway (bounded memory, whatever the line lengths). A lockfile is held
+  // to the text window too (lockfileLookbehind picks what it uses).
+  const trimRecent = () => {
+    const window = HISTORY_TEXT_CONTEXT;
+    const limit = file === null ? Infinity : sizeLimit(file);
+    while (
+      recent.length > 0 &&
+      ((recent.length > window.lines && recentChars - (recent[0].length + 1) >= window.chars) || recentChars > limit)
+    ) {
+      recentChars -= recent.shift().length + 1;
+      dropped = true;
+    }
+  };
+  const keep = (line) => {
+    textChars += line.length + 1;
+    if (file !== null && textChars > sizeLimit(file)) overflow = true;
+    if (!overflow) lines.push(line);
+  };
+
+  const flush = () => {
+    if (commit && !file && addedAny) {
+      unscanned += 1; // added lines under a header this parser could not attribute to a path
+    } else if (commit && file && addedAny) {
+      let text = lines.join('\n');
+      // UTF-16 files (and binary blobs) show up with NUL bytes; drop them so ASCII content stays scannable.
+      const hadNul = text.includes('\u0000');
+      if (hadNul) text = text.replace(/[\u0000�]/g, '');
+      const tooLarge = overflow || text.length > sizeLimit(file);
+      if ((tooLarge || hadNul) && versionIsVerifiedBinary(root, commit, file)) {
+        // A verified binary (the tree scan's own test, on the version's own first 8 KB) is skipped whatever its size: a
+        // large image must not be judged by the text size limit. Anything else, a NUL-prefixed text file included, goes on.
+        skipped.binary = (skipped.binary ?? 0) + 1;
+      } else if (tooLarge) {
+        oversize += 1;
+        unscanned += 1;
+        oversizeLimits.set(sizeLimit(file), (oversizeLimits.get(sizeLimit(file)) ?? 0) + 1);
+      } else {
+        for (const finding of scanRanges(file, text)) {
+          let touchesAddedLine = false;
+          for (let l = finding.line; l <= finding.lastLine && !touchesAddedLine; l += 1) touchesAddedLine = addedLines.has(l);
+          for (let l = finding.valueFirst; l <= finding.valueLast && !touchesAddedLine; l += 1) touchesAddedLine = addedLines.has(l);
+          if (!touchesAddedLine) continue;
+          const key = `${commit}\t${file}\t${finding.rule}`;
+          const entry = hits.get(key) ?? { commit, path: file, rule: finding.rule, count: 0 };
+          entry.count += 1;
+          hits.set(key, entry);
+        }
+      }
+    }
+    lines = [];
+    addedLines = new Set();
+    addedAny = false;
+    textChars = 0;
+    overflow = false;
+    recent = [];
+    recentChars = 0;
+    dropped = false;
+    afterLines = 0;
+    afterChars = 0;
+  };
+
+  const merges = []; // { commit, parents } of every merge commit walked (see scanMergeResults)
+  let expectParents = false;
+  const onLine = (line) => {
+    if (expectParents) {
+      // The line after each `commit ` line is `parents ...` (the format asks for it). Anything else is not output this
+      // parser understands: fail closed rather than guess which commits were merges.
+      expectParents = false;
+      const parentIds = parseParentsLine(line); // a root commit prints "parents " (a trailing blank)
+      if (parentIds === null) throw new Error('git log printed a commit header this scanner cannot parse');
+      if (parentIds.length > 1) merges.push({ commit, parents: parentIds });
+      return;
+    }
+    if (/^commit [0-9a-f]{40,64}$/.test(line)) {
+      flush();
+      commit = line.slice(7);
+      commits += 1;
+      file = null;
+      headerPath = null;
+      inHunk = false;
+      expectParents = true;
+    } else if (/^diff --(?:git|cc|combined) /.test(line)) {
+      flush();
+      file = null;
+      headerPath = diffHeaderPath(line);
+      inHunk = false;
+      parents = 1;
+    } else if (!inHunk && line.startsWith('Binary files ')) {
+      // git refused to show this version's content (for example above core.bigFileThreshold). Only a verified binary is
+      // left out (counted, as the tree scan counts it); anything else, or a path this parser cannot name, is unscanned.
+      if (headerPath !== null && versionIsVerifiedBinary(root, commit, headerPath)) skipped.binary = (skipped.binary ?? 0) + 1;
+      else unscanned += 1;
+    } else if (line.startsWith('@@')) {
+      inHunk = true;
+      parents = Math.max(1, line.match(/^@+/)[0].length - 1); // "@@@" hunks belong to 2-parent merges
+      if (lines.length > 0) keep(''); // keep lines from different hunks from looking adjacent
+      recent = [];
+      recentChars = 0;
+      dropped = false;
+      afterLines = 0;
+      afterChars = 0;
+    } else if (!inHunk) {
+      if (line.startsWith('+++ ')) {
+        const raw = line.slice(4);
+        const target = raw.startsWith('"') ? unquoteGitPath(raw) : raw.replace(/\t.*$/, '');
+        file = target === '/dev/null' ? null : target.replace(/^b\//, '');
+      }
+    } else if (line.length >= parents && !line.startsWith('\\')) {
+      const marks = line.slice(0, parents);
+      if (marks.includes('-')) return; // gone from the result
+      const content = line.slice(parents);
+      if (marks === '+'.repeat(parents)) {
+        // An added line: the context before it (up to the window), then the line itself, then the window after it.
+        let held = recent;
+        let pem = content.includes(PEM_BOUNDARY);
+        if (isLockfile(file ?? '')) {
+          const look = lockfileLookbehind(recent);
+          if (look.held.length < recent.length) dropped = true;
+          held = look.held;
+          pem = pem || look.pem;
+        }
+        if (dropped && lines.length > 0) keep('');
+        for (const line of held) keep(line);
+        recent = [];
+        recentChars = 0;
+        dropped = false;
+        keep(content);
+        addedAny = true; // an added line that is over the limit on its own still makes this version unscanned
+        if (!overflow) addedLines.add(lines.length);
+        ({ lines: afterLines, chars: afterChars } = contextFor(file ?? '', pem));
+      } else if (afterLines > 0 || afterChars > 0) {
+        keep(content);
+        afterLines -= 1;
+        afterChars -= content.length + 1;
+      } else {
+        recent.push(content);
+        recentChars += content.length + 1;
+        trimRecent();
+      }
+    }
+  };
+
+  try {
+    // Split on "\n" only. readline would also split on a lone CR and lose the continuation.
+    const decoder = new StringDecoder('utf8');
+    let pending = '';
+    let discarding = false; // inside the rest of a line that was cut at MAX_HELD_LINE_CHARS
+    for await (const chunk of child.stdout) {
+      pending += decoder.write(chunk);
+      let start = 0;
+      let newline;
+      if (discarding) {
+        newline = pending.indexOf('\n');
+        if (newline === -1) {
+          pending = '';
+          continue;
+        }
+        start = newline + 1;
+        discarding = false;
+      }
+      while ((newline = pending.indexOf('\n', start)) !== -1) {
+        onLine(pending.slice(start, newline));
+        start = newline + 1;
+      }
+      pending = pending.slice(start);
+      if (pending.length > MAX_HELD_LINE_CHARS) {
+        // One line longer than any size limit (a binary blob with few newline bytes): parse its head once and drop the
+        // rest of it, so memory stays bounded whatever the blob size. See MAX_HELD_LINE_CHARS.
+        onLine(pending.slice(0, MAX_HELD_LINE_CHARS));
+        pending = '';
+        discarding = true;
+      }
+    }
+    pending += decoder.end();
+    if (pending !== '') onLine(pending);
+    flush();
+  } catch (error) {
+    child.kill();
+    throw error;
+  }
+
+  const code = await exited;
+  if (code !== 0) throw new Error(`git log failed with exit code ${code}${summarizeStderr(stderr)}`);
+  // The merge results: what the combined diff cannot show (see scanMergeResults).
+  const merged = scanMergeResults(root, merges);
+  for (const [key, entry] of merged.hits) if (!hits.has(key)) hits.set(key, entry);
+  for (const [limit, n] of merged.oversizeLimits) oversizeLimits.set(limit, (oversizeLimits.get(limit) ?? 0) + n);
+  return { hits: [...hits.values()], commits, oversize: oversize + merged.oversize, oversizeLimits, unscanned: unscanned + merged.unscanned, skipped };
+}
+
+// ---------------------------------------------------------------------------
+// Reporting (path, line, rule, commit and counts only)
+// ---------------------------------------------------------------------------
+
+const describeRule = (id) => RULES.find((rule) => rule.id === id)?.description ?? id;
+
+/** Format tree-scan findings. The output cannot contain matched text: findings never hold it. */
+export function formatReport(findings) {
+  const lines = [
+    `check-secrets: ${findings.length} potential secret${findings.length === 1 ? '' : 's'} found (values are never printed)`,
+    '',
+  ];
+  for (const f of findings) lines.push(`  ${printable(f.path)}:${f.line}  ${f.rule}${f.source ? `  (${f.source})` : ''}`);
+  const rules = [...new Set(findings.map((f) => f.rule))];
+  lines.push('', 'Rules triggered:');
+  for (const id of rules) lines.push(`  ${id}: ${describeRule(id)}`);
+  lines.push(
+    '',
+    'If this is a real credential: remove it, rotate it (see SECURITY_NOTICE.md), do not just delete the line.',
+    `If it is a false positive: use an obvious placeholder, or add "${ALLOW_MARKER}" in a comment on that line.`,
+  );
+  return lines.join('\n');
+}
+
+const MB = 1024 * 1024;
+/** The size limit applied to a limit value, and the constant that sets it (a lockfile has its own). */
+const limitName = (bytes) => (bytes === MAX_LOCKFILE_BYTES ? 'MAX_LOCKFILE_BYTES' : 'MAX_FILE_BYTES');
+
+/** Format the text files that were too large to scan. Each path is reported against the limit that was applied to IT. */
+export function formatOversizeReport(paths) {
+  const limits = [...new Set(paths.map((p) => sizeLimit(p)))].sort((a, b) => a - b);
+  const names = limits.map(limitName);
+  const single = limits.length === 1 ? `${limits[0] / MB} MB` : 'their size limit';
+  return [
+    `check-secrets: ${paths.length} tracked text file${paths.length === 1 ? '' : 's'} over ${single} NOT scanned:`,
+    ...paths.map((p) => `  ${printable(p)}  (over ${sizeLimit(p) / MB} MB, ${limitName(sizeLimit(p))})`),
+    `A file this size cannot be checked for secrets. Split it, move it out of git, or review it by hand and raise ${names.join(' / ')}.`,
+  ].join('\n');
+}
+
+/** `oversizeLimits`: Map of size limit in bytes to the number of file versions over it. */
+const describeUnscanned = (unscanned, oversize, oversizeLimits = new Map()) => {
+  const entries = [...oversizeLimits].sort((a, b) => a[0] - b[0]);
+  const detail = entries.length === 0
+    ? `${oversize} over the ${MAX_FILE_BYTES / MB} MB limit`
+    : entries.map(([bytes, n]) => `${n} over the ${bytes / MB} MB ${bytes === MAX_LOCKFILE_BYTES ? 'lockfile ' : ''}limit`).join(', ');
+  const names = entries.map(([bytes]) => limitName(bytes));
+  return (
+    `${unscanned} file version${unscanned === 1 ? '' : 's'} NOT scanned` +
+    (oversize > 0 ? ` (${detail})` : '') +
+    ', so this audit is incomplete.' +
+    (oversize > 0 ? ` The size limit${names.length === 1 ? ' is' : 's are'} ${(names.length === 0 ? ['MAX_FILE_BYTES'] : names).join(' and ')}.` : '')
+  );
+};
+
+/** Format the tracked files that could not be read. */
+export function formatUnreadableReport(paths) {
+  return [
+    `check-secrets: ${paths.length} tracked file${paths.length === 1 ? '' : 's'} could NOT be read, so ${paths.length === 1 ? 'it was' : 'they were'} NOT scanned:`,
+    ...paths.map((p) => `  ${printable(p)}`),
+    'An unreadable file is never treated as clean. Fix its permissions or type (or remove it from git), then run again.',
+  ].join('\n');
+}
+
+/** Format history-scan hits: commit, path, rule and counts only. */
+export function formatHistoryReport(hits, { commits = 0, shallow = false, oversize = 0, unscanned = oversize, label = '--history', oversizeLimits = undefined } = {}) {
+  const lines = [];
+  if (shallow) {
+    lines.push(
+      'warning: this is a shallow clone, so only part of the history was scanned.',
+      '         Run "git fetch --unshallow" (or scan a full clone) for a complete answer.',
+      '',
+    );
+  }
+  if (unscanned > 0) lines.push(`warning: ${describeUnscanned(unscanned, oversize, oversizeLimits)}`, '');
+  const distinctCommits = new Set(hits.map((h) => h.commit)).size;
+  lines.push(
+    `check-secrets ${label}: ${hits.length} hit${hits.length === 1 ? '' : 's'} in ${distinctCommits} of ${commits} commit${commits === 1 ? '' : 's'} (values are never printed)`,
+    '',
+  );
+  for (const h of hits) {
+    lines.push(`  ${h.commit.slice(0, 7)}  ${printable(h.path)}  ${h.rule}  x${h.count}`);
+  }
+  lines.push(
+    '',
+    'Hits in history cannot be undone by deleting a line. Rotate the credential first; scrubbing history is optional.',
+  );
+  return lines.join('\n');
+}
+
+const describeSkipped = (skipped) => {
+  const entries = Object.entries(skipped);
+  const total = entries.reduce((sum, [, n]) => sum + n, 0);
+  return { total, detail: entries.length === 0 ? '' : `: ${entries.map(([reason, n]) => `${n} ${reason}`).join(', ')}` };
+};
+
+/** " (2 skipped: 2 binary)" for the history and range summaries, or nothing when no version was left out on purpose. */
+const skippedNote = (skipped) => {
+  const { total, detail } = describeSkipped(skipped);
+  return total === 0 ? '' : ` (${total} skipped${detail})`;
+};
+
+// ---------------------------------------------------------------------------
+// CLI
+// ---------------------------------------------------------------------------
+
+const USAGE = `Usage: node scripts/check-secrets.mjs [--history | --range ..]
+
+  (no flags)  scan all git-tracked text files; exit 1 on any finding
+  --history   scan added lines of every commit on every ref (owner-run, not for CI)
+  --range     scan added lines of the commits reachable from  but not from  (what CI runs on a pull
+              request or push: catches a secret committed and removed again inside the range). Needs full history:
+              a shallow clone or a commit that is not present exits 2. An all-zero  (a new branch) scans the
+               commit only.
+  --help      show this message
+
+The report lists file path, line number and rule name only. Matched text is never printed.
+`;
+
+/** Parse the command line: at most one of --history and --range .. (or --range=..). */
+function parseArguments(argv) {
+  let history = false;
+  let range = null;
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--history') history = true;
+    else if (arg === '--range' || arg.startsWith('--range=')) {
+      const value = arg === '--range' ? argv[(i += 1)] : arg.slice('--range='.length);
+      if (range !== null) return { error: '--range given twice' };
+      if (value === undefined) return { error: '--range needs a .. value' };
+      range = value;
+    } else return { error: 'unknown argument' };
+  }
+  if (history && range !== null) return { error: '--history and --range cannot be combined' };
+  return { history, range, error: null };
+}
+
+const ZERO_ID = /^0{40}(?:0{24})?$/;
+
+/** Resolve a revision to a full commit id, or null. The value is never an option: it is passed after --end-of-options. */
+function resolveCommit(root, revision) {
+  const result = spawnSync('git', ['rev-parse', '--verify', '--quiet', '--end-of-options', `${revision}^{commit}`], { cwd: root });
+  if (result.error || result.status !== 0) return null;
+  const id = result.stdout.toString('utf8').trim();
+  return /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/.test(id) ? id : null;
+}
+
+/**
+ * --range ..: scan the added lines of the commits reachable from head and not from base (`git log head --not base`,
+ * so base need not be an ancestor: a pull request is judged by what it adds, wherever its base has moved to). Same reader
+ * as --history (added-line logic, split-pair context, lockfile rules, --cc for merges) and the same fail-closed semantics:
+ * hits exit 1; an unresolvable revision, a shallow clone or an unscanned file version exit 2; a range without commits is clean.
+ * The output holds commit short ids, paths, rule names and counts, never matched text.
+ */
+async function runRange(spec, { cwd, stdout, stderr }) {
+  const parts = /^([^\s.][^\s]*?)\.\.([^\s.][^\s]*)$/.exec(spec);
+  if (!parts || parts[1].startsWith('-') || parts[1].endsWith('.') || parts[2].startsWith('-') || parts[1].includes('..') || parts[2].includes('..')) {
+    stderr.write('check-secrets --range: expected .. (two dots; a three-dot range is not accepted)\n');
+    return 2;
+  }
+  const [, baseRevision, headRevision] = parts;
+  const root = findRepoRoot(cwd);
+  if (git(['rev-parse', '--is-shallow-repository'], root).toString('utf8').trim() === 'true') {
+    stderr.write(
+      'check-secrets --range: INCOMPLETE, not a clean result. This is a shallow clone, so the commits in the range cannot be told apart from the\n' +
+        'truncated history. Fetch full history (actions/checkout fetch-depth: 0, or "git fetch --unshallow") and run again.\n',
+    );
+    return 2;
+  }
+  const head = resolveCommit(root, headRevision);
+  if (head === null) {
+    stderr.write(`check-secrets --range: INCOMPLETE. The head commit ${printable(headRevision).slice(0, 80)} is not in this repository (not fetched?), so nothing was scanned.\n`);
+    return 2;
+  }
+  const newRef = ZERO_ID.test(baseRevision);
+  let base = null;
+  if (!newRef) {
+    base = resolveCommit(root, baseRevision);
+    if (base === null) {
+      stderr.write(
+        `check-secrets --range: INCOMPLETE. The base commit ${printable(baseRevision).slice(0, 80)} is not in this repository. After a force-push the old tip is gone, and a shallow or partial fetch may not\n` +
+          'have it. Nothing was scanned, and this is not a clean result. Fetch the missing commits or check the pushed commits by hand.\n',
+      );
+      return 2;
+    }
+  }
+  const { hits, commits, oversize, oversizeLimits, unscanned, skipped } = await scanHistory(
+    root,
+    newRef ? { revisions: [head], maxCount: 1 } : { revisions: [head, '--not', base] },
+  );
+  const scope = newRef ? 'the new branch\'s tip commit only (the base is all zeros, so no earlier commit is known)' : `${commits} commit${commits === 1 ? '' : 's'} in ${baseRevision.slice(0, 12)}..${headRevision.slice(0, 12)}`;
+  if (hits.length > 0) {
+    stderr.write(`${formatHistoryReport(hits, { commits, oversize, oversizeLimits, unscanned, label: '--range' })}\n`);
+    return 1;
+  }
+  if (unscanned > 0) {
+    stderr.write(`check-secrets --range: INCOMPLETE, not a clean result (${commits} commits read, nothing found in them).\n  ${describeUnscanned(unscanned, oversize, oversizeLimits)}\n`);
+    return 2;
+  }
+  stdout.write(commits === 0 ? `check-secrets --range: no commits in ${baseRevision.slice(0, 12)}..${headRevision.slice(0, 12)}, nothing to scan\n` : `check-secrets --range: no hits in ${scope}${skippedNote(skipped)}\n`);
+  return 0;
+}
+
+/**
+ * @param {string[]} argv arguments after the script name
+ * @returns {Promise} process exit code
+ */
+export async function main(
+  argv = process.argv.slice(2),
+  { cwd = process.cwd(), stdout = process.stdout, stderr = process.stderr } = {},
+) {
+  if (argv.includes('--help') || argv.includes('-h')) {
+    stdout.write(USAGE);
+    return 0;
+  }
+  const parsed = parseArguments(argv);
+  if (parsed.error) {
+    stderr.write(`check-secrets: ${parsed.error}\n\n${USAGE}`);
+    return 2;
+  }
+
+  try {
+    if (parsed.range !== null) return await runRange(parsed.range, { cwd, stdout, stderr });
+    if (parsed.history) {
+      // Works in bare clones (git clone --mirror) too, which have no work tree.
+      let root = cwd;
+      try {
+        root = findRepoRoot(cwd);
+      } catch {
+        git(['rev-parse', '--git-dir'], cwd);
+      }
+      const shallow = git(['rev-parse', '--is-shallow-repository'], root).toString('utf8').trim() === 'true';
+      const { hits, commits, oversize, oversizeLimits, unscanned, skipped } = await scanHistory(root);
+      if (hits.length > 0) {
+        stderr.write(`${formatHistoryReport(hits, { commits, shallow, oversize, oversizeLimits, unscanned })}\n`);
+        return 1;
+      }
+      // An audit that did not look at everything is never reported as clean.
+      const gaps = [];
+      if (unscanned > 0) gaps.push(describeUnscanned(unscanned, oversize, oversizeLimits));
+      if (shallow) gaps.push('this is a shallow clone, so only part of the history was available. Run "git fetch --unshallow" or scan a full clone.');
+      if (gaps.length > 0) {
+        stderr.write(`check-secrets --history: INCOMPLETE, not a clean result (${commits} commits read, nothing found in them).\n${gaps.map((g) => `  ${g}`).join('\n')}\n`);
+        return 2;
+      }
+      stdout.write(`check-secrets --history: no hits in ${commits} commits${skippedNote(skipped)}\n`);
+      return 0;
+    }
+
+    const { findings, scanned, skipped, oversize, unreadable, fromIndex, differing } = scanTree(findRepoRoot(cwd));
+    let failed = false;
+    if (findings.length > 0) {
+      stderr.write(`${formatReport(findings)}\n`);
+      failed = true;
+    }
+    if (oversize.length > 0) {
+      stderr.write(`${formatOversizeReport(oversize)}\n`);
+      failed = true;
+    }
+    if (unreadable.length > 0) {
+      stderr.write(`${formatUnreadableReport(unreadable)}\n`);
+      failed = true;
+    }
+    if (failed) return 1;
+    const { total, detail } = describeSkipped(skipped);
+    const indexNote = fromIndex.length > 0 ? `, ${fromIndex.length} missing from the working tree and scanned from the index` : '';
+    const differNote = differing.length > 0 ? `, ${differing.length} with a staged version that differs from the working tree (both scanned)` : '';
+    stdout.write(`check-secrets: OK (${scanned} files scanned, ${total} skipped${detail}${indexNote}${differNote})\n`);
+    return 0;
+  } catch (error) {
+    stderr.write(`check-secrets: ${error.message}\n`);
+    return 2;
+  }
+}
+
+function isDirectRun() {
+  const entry = process.argv[1];
+  if (!entry) return false; // imported (or run from stdin): the caller drives main()
+  const self = fileURLToPath(import.meta.url);
+  try {
+    return realpathSync(entry) === realpathSync(self);
+  } catch {
+    // Never fall back to "not the entry point": that would exit 0 without scanning anything.
+    return path.resolve(entry) === self;
+  }
+}
+
+if (isDirectRun()) {
+  main().then((code) => {
+    process.exitCode = code;
+  });
+}