Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 8 additions & 13 deletions .github/scripts/install-public.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,9 @@ const path = require('node:path');
const os = require('node:os');
const { createHash } = require('node:crypto');
const { spawnSync } = require('node:child_process');
const lockHash = '2b3cb29398b9d148aeb8dd321e6436629c16175b42436a46fe59a71e9850825d';
const manifestHash = 'f77b805a5e292732f617c3814a24f4205ced9b326a5740e039acda6085737f34';
const lockHash = 'fed4df803f8eddce823bca0310a8227166272531baccd41d850227aede1f4d03';
const manifestHash = 'ad05ea374fd514879a01d79153b146aadc40dc5063224057e74bf4fe1eec95d1';
const approvedIntegrity = 'sha512-FISIQxV5u91Y9gvuFq3q/tA/qTqLJn3AhRcEvyozyhJTh5OBFFtv827AdHU7Rbyct4r+Ba1/99oFa1Y3yuFsdw==';
const gitSource = 'git+ssh://git@github.com/ethanresnick/negotiator.git#e2fe4abc56271b9851132f3e508e7323415c8aea';
const patchFiles = {
'LICENSE': [1172, '23d466b1b134b415b66fa50c6526b4cf3e7b9258554da88d3abb371721e7ce68'],
'index.js': [2681, '9c6c900e7e85fb599c62d9b9e4dfd2ea2f61d119dce5ed69ac3a8da828819241'],
Expand All @@ -28,7 +27,6 @@ const sha = data => createHash('sha256').update(data).digest('hex');

function verifyLock(lock) {
let publicPackages = 0;
let gitPackages = 0;
const parserPaths = [];
for (const [name, value] of Object.entries(lock.packages)) {
if (!name) continue;
Expand All @@ -37,16 +35,14 @@ function verifyLock(lock) {
assert.equal(value.integrity, approvedIntegrity);
parserPaths.push(name);
}
if (value.resolved === gitSource && name === 'node_modules/negotiator') gitPackages++;
else {
{
const url = new URL(value.resolved);
assert.equal(url.origin, 'https://registry.npmjs.org');
assert.equal(url.username + url.password + url.search + url.hash, '');
publicPackages++;
}
}
assert.equal(publicPackages, 532);
assert.equal(gitPackages, 1);
assert.deepEqual(parserPaths, ['node_modules/body-parser']);
}

Expand Down Expand Up @@ -108,12 +104,11 @@ function install() {
PATH: process.env.PATH, HOME: temporary,
npm_config_userconfig: path.join(temporary, 'npmrc'),
npm_config_globalconfig: path.join(temporary, 'global-npmrc'),
npm_config_cache: path.join(temporary, 'cache'),
GIT_CONFIG_NOSYSTEM: '1', GIT_TERMINAL_PROMPT: '0', GIT_ALLOW_PROTOCOL: 'https',
// The unchanged public Git lock source does not require an SSH credential.
GIT_CONFIG_COUNT: '1',
GIT_CONFIG_KEY_0: 'url.https://github.com/ethanresnick/negotiator.git.insteadOf',
GIT_CONFIG_VALUE_0: 'ssh://git@github.com/ethanresnick/negotiator.git'
// Job-local public cache is reused only by the offline packed-consumer test.
// GitHub removes RUNNER_TEMP with the disposable job; nothing is uploaded.
npm_config_cache: process.env.RUNNER_TEMP
? path.join(process.env.RUNNER_TEMP, 'json-api-public-cache')
: path.join(temporary, 'cache')
};
const child = spawnSync(process.execPath, [path.join(npmRoot, 'bin/npm-cli.js'), 'ci'],
{ cwd: root, env, stdio: 'inherit', timeout: 240000 });
Expand Down
4 changes: 2 additions & 2 deletions .github/scripts/install-public.test.cjs
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ test('rejects private or authenticated registry destinations before npm runs', (
assert.throws(() => verifyLock(lock));
}
});
test('rejects changed Git source identity before npm runs', () => {
const lock = fresh(); lock.packages['node_modules/negotiator'].resolved += '0';
test('rejects every external Git source before npm runs', () => {
const lock = fresh(); lock.packages['node_modules/body-parser'].resolved = 'git+https://github.com/ethanresnick/negotiator.git#e2fe4abc56271b9851132f3e508e7323415c8aea';
assert.throws(() => verifyLock(lock));
});

Expand Down
1 change: 1 addition & 0 deletions .github/scripts/qualify.sh
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ export npm_config_audit=false npm_config_fund=false npm_config_ignore_scripts=tr
unset NODE_OPTIONS NODE_PATH

timeout --kill-after=5s 180s npm run build
timeout --kill-after=5s 120s npm run test:vendor
if [ "$1" = full ]; then
timeout --kill-after=5s 120s npm run test:tooling
else
Expand Down
8 changes: 8 additions & 0 deletions .github/workflows/qualification.yml
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,10 @@ jobs:
run: node .github/scripts/install-public.cjs
- name: Build, lint, tooling and both unit forms
run: bash .github/scripts/qualify.sh full
- name: Verify isolated packed consumer offline
env:
JSON_API_PUBLIC_CACHE: ${{ runner.temp }}/json-api-public-cache
run: timeout --kill-after=5s 180s npm run test:package
- name: Complete isolated database integration suite
env:
HOST: 127.0.0.1
Expand Down Expand Up @@ -78,3 +82,7 @@ jobs:
package-manager-cache: false
- name: Build, lint and runtime compatibility
run: bash .github/scripts/qualify.sh compatibility
- name: Verify isolated packed consumer offline
env:
JSON_API_PUBLIC_CACHE: ${{ runner.temp }}/json-api-public-cache
run: timeout --kill-after=5s 180s npm run test:package
21 changes: 18 additions & 3 deletions docs/ci-qualification.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,32 @@ the emitted public entry, fixture installation, and the complete existing unit
and integration selection. Node 18.20.8 runs lint/build, source and compiled
units, the public entry, body-parser and lint controls. Dependencies are installed
under the Node 24 distribution's npm 11.19.1 before switching the compatibility
job to Node 18. No runtime, package manifest, or dependency lock changes are made.
job to Node 18. The jobs do not rewrite runtime source, manifests or the lock.
Both runtimes also run the original 250 internal Negotiator tests, bounded parser
regressions, and an actual offline packed-package consumer installation.

Installation uses an empty temporary credential route, disabled install scripts
and audit requests, the public npm registry, and a seven-day release-age guard.
The existing exact body-parser 1.20.9 exception is bounded by the frozen source
hashes, exact version/integrity, and all ten installed tarball file hashes.
The unchanged public Negotiator commit uses an invocation-only HTTPS rewrite;
no SSH keys, global configuration, private grants, or private registries are used.
All 532 external lock nodes use the public registry. The custom Negotiator runtime
is now internal source under `vendor/negotiator`, with MIT/provenance retained and
the exact upstream language-parser fix. External Git dependencies and their
HTTPS rewrite exception are removed. No SSH keys, global configuration, private
grants or private registries are used.
Dependency changes require a renewed source/age review and updated qualification
hashes. A successful workflow is not a fresh vulnerability scan.

The install retains only its public npm cache in the disposable job's
`RUNNER_TEMP`; no cache is uploaded or shared between jobs. The package test uses
that cache offline, with lifecycle scripts disabled, and rejects dependency
versions outside the reviewed lock. It packs the real library using its existing
prepare build, installs it with lifecycle scripts disabled in a
temporary downstream project, verifies source/compiled internal runtime bytes,
exercises the public entry and real media/language behavior, and compiles a
declaration consumer. Checkout fallback is rejected. Its isolated consumer is
removed on completion; GitHub removes the job cache with the disposable job.

The integration job uses only its disposable official Mongo 7.0.45 Jammy service,
pinned by digest and mapped to loopback port 27017. Existing tests hard-code
`mongodb://localhost/integration-test`; IPv4 resolution and the HTTP test host
Expand Down
10 changes: 0 additions & 10 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 12 additions & 4 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@
"depd": "^1.1.2",
"immutable": "^3.8.4",
"lodash": "^4.18.1",
"negotiator": "git+https://github.com/ethanresnick/negotiator.git#e2fe4abc56271b9851132f3e508e7323415c8aea",
"pluralize": "0.0.11",
"pug": "^3.0.3",
"qs": "^6.5.3",
Expand Down Expand Up @@ -113,15 +112,17 @@
"test:unit": "NODE_ENV=testing mocha --require ts-node/register --extension ts --recursive test/unit/ --full-trace --check-leaks",
"test:tooling": "node --test test/tooling-security.cjs test/lint-compatibility.cjs test/body-parser-security.cjs",
"test:debug": "npm run test -- --inspect-brk",
"build:lintfree": "npm run clean && cpr templates build/templates && echo Using TypeScript && tsc --version && tsc --pretty",
"build:lintfree": "npm run clean && cpr templates build/templates && cpr vendor build/vendor && echo Using TypeScript && tsc --version && tsc --pretty",
"build": "npm run lint && npm run build:lintfree",
"prepare": "npm run build",
"prepublishOnly": "npm run test",
"cover": "NODE_ENV=test npm run test:install-fixtures && nyc npm run test:fixtures-installed",
"cover:view": "npm run cover && open coverage/index.html",
"watch": "npm run build -- --watch",
"watch:lintfree": "npm run build:lintfree -- --watch",
"watch:test": "npm run test -- --watch"
"watch:test": "npm run test -- --watch",
"test:vendor": "node --test --test-reporter=tap test/negotiator-security.cjs && mocha --reporter dot --check-leaks test/vendor-negotiator/",
"test:package": "node scripts/verify-packed-consumer.cjs"
},
"bugs": {
"url": "https://github.com/ethanresnick/json-api/issues"
Expand All @@ -138,5 +139,12 @@
],
"engines": {
"node": ">=18.20.8"
}
},
"files": [
"build",
"src",
"typings",
"templates",
"vendor"
]
}
127 changes: 127 additions & 0 deletions scripts/verify-packed-consumer.cjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
'use strict';

// Local/offline package delivery proof. The cache must contain only the reviewed
// public lock's packages; no credentials, registry requests or install scripts.
// Packing runs this repository's existing prepare build, as publication would.
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const root = path.resolve(__dirname, '..');
const cache = process.env.JSON_API_PUBLIC_CACHE;
assert(cache && path.isAbsolute(cache), 'Set JSON_API_PUBLIC_CACHE to the reviewed temporary public cache');
const npm = path.resolve(path.dirname(fs.realpathSync(process.execPath)), '../lib/node_modules/npm/bin/npm-cli.js');
const lock = JSON.parse(fs.readFileSync(path.join(root, 'package-lock.json')));
const manifest = JSON.parse(fs.readFileSync(path.join(root, 'package.json')));
const reviewed = new Set(Object.entries(lock.packages).filter(([location]) => location).map(([location, item]) =>
(item.name || location.split('node_modules/').pop()) + '@' + item.version));
const temporary = fs.mkdtempSync(path.join(os.tmpdir(), 'json-api-packed-consumer-'));
try {
const userconfig = path.join(temporary, 'npmrc');
fs.writeFileSync(userconfig, 'registry=https://registry.npmjs.org\naudit=false\nfund=false\nignore-scripts=true\noffline=true\nmin-release-age=7\nmin-release-age-exclude[]=body-parser\n');
const env = { PATH: process.env.PATH, HOME: temporary, npm_config_userconfig: userconfig,
npm_config_globalconfig: '/dev/null', npm_config_cache: cache };
function run(args, cwd) {
const child = spawnSync(process.execPath, args, { cwd, env, encoding: 'utf8', timeout: 120000 });
assert.ifError(child.error);
assert.equal(child.status, 0, (child.stdout + child.stderr).slice(-4000));
return child.stdout;
}
const packed = JSON.parse(run([npm, 'pack', '--ignore-scripts=false', '--foreground-scripts=false', '--json', '--pack-destination', temporary], root))[0];
const files = new Set(packed.files.map(file => file.path));
for (const name of ['index.js', 'index.d.ts', 'lib/language.js', 'lib/mediaType.js', 'LICENSE']) {
assert(files.has('vendor/negotiator/' + name));
assert(files.has('build/vendor/negotiator/' + name));
}
assert(files.has('build/src/index.js') && files.has('build/src/index.d.ts'));
assert(files.has('src/steps/http/content-negotiation/negotiate-content-type.ts'));
assert(![...files].some(name => name.startsWith('node_modules/') || name.startsWith('.github/')));
const consumer = path.join(temporary, 'consumer'); fs.mkdirSync(consumer);
const tarball = path.join(temporary, packed.filename);
const consumerManifest = { name: 'isolated-public-consumer', version: '1.0.0', private: true,
dependencies: { 'json-api': 'file:' + tarball, express: lock.packages['node_modules/express'].version,
mongoose: lock.packages['node_modules/mongoose'].version } };
fs.writeFileSync(path.join(consumer, 'package.json'), JSON.stringify(consumerManifest));
// A clean downstream install uses a lock, just as a real reproducible consumer
// should. Project only its reachable nodes from the reviewed library lock.
// npm ci's cache contains exact tarballs, not registry version-selection data.
const packages = { '': consumerManifest, 'node_modules/json-api': {
version: manifest.version, resolved: 'file:' + tarball, integrity: packed.integrity,
dependencies: manifest.dependencies, peerDependencies: manifest.peerDependencies,
engines: manifest.engines
} };
function resolve(name, from) {
let current = from;
while (true) {
const location = (current ? current + '/' : '') + 'node_modules/' + name;
if (lock.packages[location]) return location;
if (!current) return undefined;
const parent = path.posix.dirname(current);
current = parent === '.' ? '' : parent;
}
}
const queue = Object.keys(manifest.dependencies).concat(['express', 'mongoose']).map(name => resolve(name, ''));
while (queue.length) {
const location = queue.shift(); assert(location, 'Missing reviewed dependency');
if (packages[location]) continue;
const item = { ...lock.packages[location] }; delete item.dev; delete item.devOptional;
packages[location] = item;
for (const name of Object.keys({ ...item.dependencies, ...item.optionalDependencies, ...item.peerDependencies })) {
const next = resolve(name, location);
if (next) queue.push(next);
else assert(item.optionalDependencies?.[name] || item.peerDependenciesMeta?.[name]?.optional, 'Missing reviewed transitive dependency: ' + name);
}
}
fs.writeFileSync(path.join(consumer, 'package-lock.json'), JSON.stringify({ name: consumerManifest.name,
version: consumerManifest.version, lockfileVersion: 3, requires: true, packages }));
run([npm, 'ci', '--offline', '--ignore-scripts', '--legacy-peer-deps', '--no-audit', '--no-fund'], consumer);
const installed = JSON.parse(fs.readFileSync(path.join(consumer, 'package-lock.json')));
for (const [location, item] of Object.entries(installed.packages)) {
if (!location || location === 'node_modules/json-api') continue;
const name = item.name || location.split('node_modules/').pop();
assert(reviewed.has(name + '@' + item.version), 'Unreviewed downstream resolution: ' + name);
assert.equal(new URL(item.resolved).origin, 'https://registry.npmjs.org');
if (name === 'body-parser') assert.equal(item.version, '1.20.9');
}
const packageRoot = path.join(consumer, 'node_modules/json-api');
assert(!JSON.parse(fs.readFileSync(path.join(packageRoot, 'package.json'))).dependencies.negotiator);
for (const name of ['index.js', 'lib/charset.js', 'lib/encoding.js', 'lib/language.js', 'lib/mediaType.js']) {
const original = fs.readFileSync(path.join(root, 'vendor/negotiator', name));
assert.deepEqual(fs.readFileSync(path.join(packageRoot, 'vendor/negotiator', name)), original);
assert.deepEqual(fs.readFileSync(path.join(packageRoot, 'build/vendor/negotiator', name)), original);
}
const smoke = `
const assert=require('node:assert/strict');
const api=require('json-api');
const N=require('json-api/build/vendor/negotiator');
const sourceN=require('json-api/vendor/negotiator');
const negotiate=require('json-api/build/src/steps/http/content-negotiation/negotiate-content-type').default;
assert.equal(typeof api.APIController,'function');
for(const Constructor of [N,sourceN]){
assert.deepEqual(new Constructor({headers:{'accept-language':'en-US,en;q=0.8'}}).languages(),['en-US','en']);
const rows=new Constructor({headers:{accept:'application/vnd.api+json;ext=example,application/json;q=0.8'}}).mediaTypes(undefined,{detailed:true});
assert.equal(rows[0].type,'application/vnd.api+json');assert.equal(rows[0].parameters.ext,'example');
assert.equal(Object.getPrototypeOf(rows[0].parameters),null);
}
(async()=>{
assert.equal(await negotiate('application/vnd.api+json',['application/vnd.api+json']),'application/vnd.api+json');
await assert.rejects(negotiate('application/vnd.api+json;ext=x,application/json',['application/vnd.api+json']),e=>e.status==='406');
assert.equal(await negotiate('application/vnd.api+json;ext=x,application/vnd.api+json',['application/vnd.api+json']),'application/vnd.api+json');
const prefix=process.cwd()+require('node:path').sep+'node_modules'+require('node:path').sep;
assert(Object.keys(require.cache).every(file=>file.startsWith(prefix)),'Checkout fallback loaded');
console.log('PACKED_PUBLIC_ENTRY_MEDIA_LANGUAGE_PASS');
})().catch(e=>{console.error(e);process.exitCode=1;});`;
assert(run(['-e', smoke], consumer).includes('PACKED_PUBLIC_ENTRY_MEDIA_LANGUAGE_PASS'));
const child = spawnSync(process.execPath, ['-e', "const a=require('node:assert/strict'),N=require('json-api/build/vendor/negotiator');a.deepEqual(new N({headers:{'accept-language':'a-'.repeat(30000)+'x y'}}).languages(),[]);"],
{ cwd: consumer, env, timeout: 1000, killSignal: 'SIGKILL' });
assert.ifError(child.error); assert.equal(child.status, 0);
fs.writeFileSync(path.join(consumer, 'consumer.ts'), 'import N = require("json-api/build/vendor/negotiator");\nconst rows: N.DetailedMediaType[] = new N({headers:{accept:"application/json"}}).mediaTypes(undefined,{detailed:true});\nconst languages: string[] = new N({headers:{}}).languages();\nconsole.log(rows,languages);\n');
run([path.join(root, 'node_modules/typescript/bin/tsc'), '--noEmit', '--strict', '--skipLibCheck', '--target', 'es2018', '--module', 'commonjs', '--moduleResolution', 'node', 'consumer.ts'], consumer);
console.log(JSON.stringify({ state: 'PACKED_ISOLATED_OFFLINE_CONSUMER_PASS', files: files.size,
packageIntegrity: packed.integrity, installedNodes: Object.keys(installed.packages).length - 1,
checkoutFallback: false, sourceAndCompiledVendorBytesMatch: true, declarationsPass: true,
libraryVersion: packed.version, dependencyVersionsRestrictedToReviewedLock: true }));
} finally {
fs.rmSync(temporary, { recursive: true, force: true });
}
2 changes: 1 addition & 1 deletion src/controllers/Documentation.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import _ = require("lodash");
import path = require("path");
import pug = require("pug");
import Negotiator = require("negotiator");
import Negotiator = require("../../vendor/negotiator");
import dasherize = require("dasherize");

import ResourceTypeRegistry, { ResourceTypeDescription, ResourceTypeInfo } from "../ResourceTypeRegistry";
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import Negotiator = require("negotiator");
import Negotiator = require("../../../../vendor/negotiator");
import * as Errors from "../../../util/errors";
import { objectIsEmpty } from "../../../util/misc";

Expand Down
Loading
Loading