From 7e10b997a6b685366ef773b505e1582c4cb12382 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 6 Oct 2026 16:29:40 -0700 Subject: [PATCH 1/5] fix(test): collect coverage before loader transforms Signed-off-by: Prekshi Vyas --- package-lock.json | 413 ++++++++++++++++- package.json | 3 + .../gateway-route-mutation-lock.test.ts | 22 + .../sandbox-host-local-reservation.test.ts | 39 +- test/helpers/onboard-script-mocks.cjs | 1 + test/helpers/register-source-require.ts | 404 +---------------- test/helpers/source-coverage-plugin.ts | 26 ++ test/helpers/source-coverage-provider.mts | 18 + test/helpers/source-coverage-report.mts | 120 +++++ test/helpers/source-coverage.cts | 192 ++++++++ test/helpers/source-require-compiler.ts | 415 ++++++++++++++++++ test/repository/source-coverage.test.ts | 317 +++++++++++++ test/repository/source-require-loader.test.ts | 11 +- vitest.config.ts | 10 +- 14 files changed, 1574 insertions(+), 417 deletions(-) create mode 100644 test/helpers/source-coverage-plugin.ts create mode 100644 test/helpers/source-coverage-provider.mts create mode 100644 test/helpers/source-coverage-report.mts create mode 100644 test/helpers/source-coverage.cts create mode 100644 test/helpers/source-require-compiler.ts create mode 100644 test/repository/source-coverage.test.ts diff --git a/package-lock.json b/package-lock.json index 0fa49d48f9e..5f0f5907df9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -35,6 +35,7 @@ "nemohermes": "bin/nemohermes.js" }, "devDependencies": { + "@babel/core": "^7.29.7", "@commitlint/cli": "^20.5.0", "@commitlint/config-conventional": "^20.5.0", "@earendil-works/pi-coding-agent": "0.80.6", @@ -44,9 +45,11 @@ "convert-source-map": "2.0.0", "eslint-plugin-sonarjs": "4.2.0", "fast-check": "^4.8.0", + "istanbul-lib-instrument": "6.0.3", "oxfmt": "0.63.0", "oxlint": "1.78.0", "oxlint-tsgolint": "7.0.2001", + "picomatch": "^4.0.4", "tsx": "^4.21.0", "typescript": "6.0.3", "vitest": "^4.1.9" @@ -595,13 +598,13 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.29.0", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.0.tgz", - "integrity": "sha512-9NhCeYjq9+3uxgdtp20LSiJXJvN0FeCtNGpJxuMFZ1Kv3cWUNb6DOhJwUvcVCzKGR66cw4njwM6hrJLqgOwbcw==", + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", "dev": true, "license": "MIT", "dependencies": { - "@babel/helper-validator-identifier": "^7.28.5", + "@babel/helper-validator-identifier": "^7.29.7", "js-tokens": "^4.0.0", "picocolors": "^1.1.1" }, @@ -609,6 +612,153 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/core/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/@babel/helper-compilation-targets/node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, "node_modules/@babel/helper-string-parser": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.7.tgz", @@ -629,6 +779,30 @@ "node": ">=6.9.0" } }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.7.tgz", + "integrity": "sha512-1k2lAGRMfHTcwuNYcCNUmaUffmQv8KWMfh2iJUUeRlwlwH4FdNG7mfPI10NPfLHJFThE4Tyr4mv7kTNZOiPuBg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/parser": { "version": "7.29.8", "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.8.tgz", @@ -645,6 +819,40 @@ "node": ">=6.0.0" } }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.8.tgz", + "integrity": "sha512-I5z7H3bf/41ktsNVLtpN0wAa336HkqIHQ5BuPLEhTkt1jVSyZpeNKIzTgEWmlxjdg81R0IgUCcaE+Ok3NvrfZg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.8", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/@babel/types": { "version": "7.29.8", "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.8.tgz", @@ -1519,7 +1727,7 @@ "typebox": "1.1.38" }, "bin": { - "pi-ai": "dist/cli.js" + "pi-ai": "./dist/cli.js" }, "engines": { "node": ">=22.19.0" @@ -3620,6 +3828,16 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@istanbuljs/schema": { + "version": "0.1.6", + "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.6.tgz", + "integrity": "sha512-+Sg6GCR/wy1oSmQDFq4LQDAhm3ETKnorxN+y5nbLULOR3P0c14f2Wurzj3/xqPXtasLFfHd5iRFQ7AJt4KH2cw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + } + }, "node_modules/@j178/prek": { "version": "0.3.6", "resolved": "https://registry.npmjs.org/@j178/prek/-/prek-0.3.6.tgz", @@ -3662,6 +3880,28 @@ "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, "node_modules/@jridgewell/resolve-uri": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", @@ -5402,6 +5642,19 @@ "node": "18 || 20 || >=22" } }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.27", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.27.tgz", + "integrity": "sha512-ElY12DaROGuan+lMmZ8Cvo/ZUbXPe7Enc/9VU/b1T3Kp4dwytRcNdR8DoSJN5SNJT/CuvcCA0DHDVmMOCePdRQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/bowser": { "version": "2.14.1", "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", @@ -5420,6 +5673,40 @@ "node": "20 || >=22" } }, + "node_modules/browserslist": { + "version": "4.29.3", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.3.tgz", + "integrity": "sha512-1R4kiYKXGViqEN0CnoDrXc1StD9niAwu+j2dukWzrD4bJgsD4lDmEp0CRbc6E/vYJIfTHwPmwyaKtVSudICdPA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.11.26", + "caniuse-lite": "^1.0.30001813", + "electron-to-chromium": "^1.5.439", + "node-releases": "^2.0.57", + "update-browserslist-db": "^1.3.3" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, "node_modules/builtin-modules": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-3.3.0.tgz", @@ -5467,6 +5754,27 @@ "node": ">=6" } }, + "node_modules/caniuse-lite": { + "version": "1.0.30001814", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001814.tgz", + "integrity": "sha512-/Uaf1lAzr59XcMpW0o96WoEfr+VXK2OX4U9AgFoiSHsVJ4HppnIFUjtYzsyDH2+tgANaQb2/oxYGwCPapN1FpA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, "node_modules/chai": { "version": "6.2.2", "resolved": "https://registry.npmjs.org/chai/-/chai-6.2.2.tgz", @@ -5806,6 +6114,13 @@ "node": ">=0.10.0" } }, + "node_modules/electron-to-chromium": { + "version": "1.5.444", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.444.tgz", + "integrity": "sha512-5ss/uJfoDYDHT0lfJzT6FbcskIzROIOPf0BbbFkGcvDzoJU7i//9GDrwwIHQVmIsrAGiF3ihpADBRIsrEFt1rQ==", + "dev": true, + "license": "ISC" + }, "node_modules/emoji-regex": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", @@ -6556,6 +6871,16 @@ "dev": true, "license": "MIT" }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, "node_modules/get-caller-file": { "version": "2.0.5", "resolved": "https://registry.npmjs.org/get-caller-file/-/get-caller-file-2.0.5.tgz", @@ -6989,6 +7314,23 @@ "node": ">=8" } }, + "node_modules/istanbul-lib-instrument": { + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/istanbul-lib-instrument/-/istanbul-lib-instrument-6.0.3.tgz", + "integrity": "sha512-Vtgk7L/R2JHyyGW07spoFlB8/lpjiOLTjMdms6AFMraYt3BaJauod/NGrfnVG/y4Ix1JEuMRPDPEj2ua+zz1/Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "@babel/core": "^7.23.9", + "@babel/parser": "^7.23.9", + "@istanbuljs/schema": "^0.1.3", + "istanbul-lib-coverage": "^3.2.0", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/istanbul-lib-report": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/istanbul-lib-report/-/istanbul-lib-report-3.0.1.tgz", @@ -7097,6 +7439,19 @@ "js-yaml": "bin/js-yaml.js" } }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/json-buffer": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz", @@ -7643,6 +7998,16 @@ "license": "MIT", "peer": true }, + "node_modules/node-releases": { + "version": "2.0.57", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.57.tgz", + "integrity": "sha512-kQK9LGGFiHtrWiNhZtA7Qbw17AQz+dmsEKODRIVTXA9+e5MS/2gZEBhYJt13GrAz5/IOZKddH/0Z3TP/Zgo+yw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/npm-run-path": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/npm-run-path/-/npm-run-path-6.0.0.tgz", @@ -8542,6 +8907,37 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/update-browserslist-db": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.3.tgz", + "integrity": "sha512-pJ2sYawQS0R/WI928Gj5GlPhTGzbMelq0+4INtSYNDV9ErKJcX6xjGWkoG/VnB3dpUm00zALaqkrUD77pO5TDQ==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, "node_modules/uri-js": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/uri-js/-/uri-js-4.4.1.tgz", @@ -8842,6 +9238,13 @@ "node": ">=10" } }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + }, "node_modules/yaml": { "version": "2.8.3", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", diff --git a/package.json b/package.json index 99fc3f04759..d4220fa4289 100644 --- a/package.json +++ b/package.json @@ -165,6 +165,7 @@ "url": "https://github.com/NVIDIA/NemoClaw.git" }, "devDependencies": { + "@babel/core": "^7.29.7", "@commitlint/cli": "^20.5.0", "@commitlint/config-conventional": "^20.5.0", "@earendil-works/pi-coding-agent": "0.80.6", @@ -174,9 +175,11 @@ "convert-source-map": "2.0.0", "eslint-plugin-sonarjs": "4.2.0", "fast-check": "^4.8.0", + "istanbul-lib-instrument": "6.0.3", "oxfmt": "0.63.0", "oxlint": "1.78.0", "oxlint-tsgolint": "7.0.2001", + "picomatch": "^4.0.4", "tsx": "^4.21.0", "typescript": "6.0.3", "vitest": "^4.1.9" diff --git a/src/lib/inference/gateway-route-mutation-lock.test.ts b/src/lib/inference/gateway-route-mutation-lock.test.ts index 3c985b44e26..9ec7fcb3001 100644 --- a/src/lib/inference/gateway-route-mutation-lock.test.ts +++ b/src/lib/inference/gateway-route-mutation-lock.test.ts @@ -31,6 +31,28 @@ function observeNextLockPublicationAttempt(): { attempted: Promise; restor } describe("gateway route mutation lock", () => { + it("rejects blank gateway names before running sync or async mutations", () => { + const operation = vi.fn(); + expect(() => withGatewayRouteMutationLock(" \t", operation)).toThrow( + "OpenShell gateway name is required.", + ); + expect(() => withGatewayRouteMutationLockSync(" \t", operation)).toThrow( + "OpenShell gateway name is required.", + ); + expect(operation).not.toHaveBeenCalled(); + }); + + it.each([0, 65_536, 1.5, Number.NaN])( + "rejects invalid router port %s before running a lifecycle mutation", + (port) => { + const operation = vi.fn(); + expect(() => withModelRouterPortLifecycleLock(port, operation)).toThrow( + "Model Router port must be an integer from 1 to 65535.", + ); + expect(operation).not.toHaveBeenCalled(); + }, + ); + it("resolves default and overridden router lock directories without filesystem access", () => { const homeDir = "/srv/nemoclaw-controller"; const expectedStateDir = path.join(homeDir, ".nemoclaw", "state"); diff --git a/src/lib/onboard/machine/handlers/sandbox-host-local-reservation.test.ts b/src/lib/onboard/machine/handlers/sandbox-host-local-reservation.test.ts index f9790e813ba..2292e19d15b 100644 --- a/src/lib/onboard/machine/handlers/sandbox-host-local-reservation.test.ts +++ b/src/lib/onboard/machine/handlers/sandbox-host-local-reservation.test.ts @@ -3,9 +3,7 @@ import assert from "node:assert/strict"; import fs from "node:fs/promises"; -import os from "node:os"; -import path from "node:path"; -import { afterEach, beforeEach, expect, it, vi } from "vitest"; +import { afterAll, afterEach, beforeEach, expect, it, vi } from "vitest"; import { serializedLlamaCppHostLocalInferenceReceipt } from "../../../../../test/helpers/host-local-inference-receipt"; vi.mock("../../messaging-channel-setup", () => ({ @@ -13,27 +11,34 @@ vi.mock("../../messaging-channel-setup", () => ({ detectUnconfiguredMessagingChannels: vi.fn(() => []), })); -let home: string; +const home = await vi.hoisted(async () => { + const fs = await import("node:fs/promises"); + const os = await import("node:os"); + const path = await import("node:path"); + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-host-local-resume-")); + vi.stubEnv("HOME", directory); + return directory; +}); + +import { handleSandboxState } from "./sandbox"; +import { baseOptions, createDeps } from "./sandbox-test-fixtures"; +import { createSession } from "../../../state/onboard-session"; +import * as registry from "../../../state/registry"; +import { normalizeInferenceSelection } from "../../../inference/selection"; +import { qualifyPendingSandboxCreateReservation } from "../../../state/registry/route-reservation"; +import { createSandboxHostLocalInferenceProvenance } from "../../../state/registry/host-local-inference"; + beforeEach(async () => { - home = await fs.mkdtemp(path.join(os.tmpdir(), "nemoclaw-host-local-resume-")); - vi.stubEnv("HOME", home); - vi.resetModules(); + await fs.mkdir(home, { recursive: true }); }); afterEach(async () => { - vi.unstubAllEnvs(); await fs.rm(home, { recursive: true, force: true }); }); +afterAll(() => { + vi.unstubAllEnvs(); +}); async function resumedHostLocalSandbox() { - const { handleSandboxState } = await import("./sandbox"); - const { baseOptions, createDeps } = await import("./sandbox-test-fixtures"); - const { createSession } = await import("../../../state/onboard-session"); - const registry = await import("../../../state/registry"); - const { normalizeInferenceSelection } = await import("../../../inference/selection"); - const { qualifyPendingSandboxCreateReservation } = - await import("../../../state/registry/route-reservation"); - const { createSandboxHostLocalInferenceProvenance } = - await import("../../../state/registry/host-local-inference"); const session = createSession({ sandboxName: "saved" }); session.steps.sandbox.status = "complete"; const hostLocalInferenceReceipt = serializedLlamaCppHostLocalInferenceReceipt("docker"); diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index f4912f118f3..95751ac27f5 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -98,6 +98,7 @@ function registerSourceRequire() { const bootstrapTypeScriptFiles = new Set([ path.resolve(sourceLoader), path.resolve(__dirname, "source-require-cache.ts"), + path.resolve(__dirname, "source-require-compiler.ts"), ]); const previousTypeScriptLoader = Module._extensions[".ts"]; diff --git a/test/helpers/register-source-require.ts b/test/helpers/register-source-require.ts index 12473d19a9e..38ee364f961 100644 --- a/test/helpers/register-source-require.ts +++ b/test/helpers/register-source-require.ts @@ -1,32 +1,16 @@ // SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. // SPDX-License-Identifier: Apache-2.0 -import crypto from "node:crypto"; import fs from "node:fs"; import Module from "node:module"; -import os from "node:os"; import path from "node:path"; -import { performance } from "node:perf_hooks"; -import ts from "typescript"; - -import { - loadSourceRequireCompilerOptions, - sourceRequireCacheDir, - sourceRequireCachePath, -} from "./source-require-cache"; +import { instrumentSourceForCoverage, shouldInstrumentSource } from "./source-coverage.cts"; +import { compileSourceRequire } from "./source-require-compiler"; type CommonJsModule = NodeModule & { _compile(source: string, filename: string): void; }; -type LockSnapshot = { - contents: string; - dev: number; - ino: number; - mtimeMs: number; - size: number; -}; - type ResolveFilename = ( request: string, parent?: CommonJsModule | null, @@ -39,373 +23,6 @@ const moduleRuntime = Module as unknown as { _resolveFilename: ResolveFilename; }; const repoRoot = path.resolve(__dirname, "../.."); -const compilerOptions = loadSourceRequireCompilerOptions(repoRoot); -// Keep the cross-process transpilation cache in this checkout's dependency -// tree. A shared, predictable directory under the OS temp root could be -// replaced by another local user before a test process reads from it. -const cacheDir = sourceRequireCacheDir(repoRoot); -fs.mkdirSync(cacheDir, { recursive: true }); -const cacheWaitMs = envInt("NEMOCLAW_SOURCE_REQUIRE_CACHE_WAIT_MS", 5_000); -const cachePollMs = Math.max(1, envInt("NEMOCLAW_SOURCE_REQUIRE_CACHE_POLL_MS", 25)); -const cacheLockStaleMs = envInt("NEMOCLAW_SOURCE_REQUIRE_CACHE_LOCK_STALE_MS", 30_000); -const statsPath = sourceRequireStatsPath(process.env.NEMOCLAW_SOURCE_REQUIRE_STATS); - -const stats = { - cacheHits: 0, - cacheMisses: 0, - compileMs: 0, - duplicateFallbacks: 0, - files: 0, - lockWaits: 0, - readCacheMs: 0, - staleLocks: 0, - transforms: 0, - transformMs: 0, - waitMs: 0, -}; - -function envInt(name: string, fallback: number): number { - const raw = process.env[name]; - if (raw === undefined) return fallback; - const parsed = Number.parseInt(raw, 10); - return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; -} - -function isWithin(root: string, candidate: string): boolean { - const relative = path.relative(root, candidate); - return ( - relative === "" || - (!path.isAbsolute(relative) && !relative.startsWith(`..${path.sep}`) && relative !== "..") - ); -} - -function sourceRequireStatsPath(configuredPath: string | undefined): string | undefined { - if (!configuredPath) return undefined; - const resolved = path.resolve(configuredPath); - return [repoRoot, os.tmpdir()].some((root) => isWithin(path.resolve(root), resolved)) - ? resolved - : undefined; -} - -function openSourceRequireStats(statsPath: string): number | null { - const parent = path.dirname(statsPath); - let canonicalParent: string; - try { - canonicalParent = fs.realpathSync.native(parent); - } catch { - return null; - } - const canonicalPath = path.join(canonicalParent, path.basename(statsPath)); - const withinAllowedRoot = [repoRoot, os.tmpdir()].some((root) => { - try { - return isWithin(fs.realpathSync.native(root), canonicalPath); - } catch { - return false; - } - }); - if (!withinAllowedRoot) return null; - - let descriptor: number; - try { - descriptor = fs.openSync( - statsPath, - fs.constants.O_APPEND | - fs.constants.O_CREAT | - fs.constants.O_WRONLY | - fs.constants.O_NOFOLLOW, - 0o600, - ); - } catch { - return null; - } - const stat = fs.fstatSync(descriptor); - if (!stat.isFile() || stat.nlink !== 1) { - fs.closeSync(descriptor); - return null; - } - return descriptor; -} - -function nowMs(): number { - return performance.now(); -} - -const sleepBuffer = new SharedArrayBuffer(4); -const sleepArray = new Int32Array(sleepBuffer); - -function sleepSync(ms: number): void { - if (ms <= 0) return; - Atomics.wait(sleepArray, 0, 0, ms); -} - -function readCachedOutput(cachePath: string): string | null { - const start = nowMs(); - try { - const output = fs.readFileSync(cachePath, "utf8"); - stats.readCacheMs += nowMs() - start; - return output; - } catch (error) { - stats.readCacheMs += nowMs() - start; - if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; - return null; - } -} - -function writeAtomic(cachePath: string, outputText: string): void { - const temporaryPath = `${cachePath}.${process.pid}.${crypto.randomUUID()}`; - fs.writeFileSync(temporaryPath, outputText, { flag: "wx", mode: 0o600 }); - try { - fs.renameSync(temporaryPath, cachePath); - } catch (error) { - fs.rmSync(temporaryPath, { force: true }); - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - } -} - -function readLockSnapshot(lockPath: string): LockSnapshot | null { - let descriptor: number; - try { - descriptor = fs.openSync(lockPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code === "ENOENT" || code === "ELOOP") return null; - throw error; - } - - try { - const before = fs.fstatSync(descriptor); - const contents = fs.readFileSync(descriptor, "utf8"); - const after = fs.fstatSync(descriptor); - if ( - before.dev !== after.dev || - before.ino !== after.ino || - before.mtimeMs !== after.mtimeMs || - before.size !== after.size - ) { - return null; - } - return { - contents, - dev: after.dev, - ino: after.ino, - mtimeMs: after.mtimeMs, - size: after.size, - }; - } finally { - fs.closeSync(descriptor); - } -} - -function parseLockOwner(contents: string): { pid?: number } { - try { - const parsed = JSON.parse(contents); - return typeof parsed?.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 - ? { pid: parsed.pid } - : {}; - } catch { - return {}; - } -} - -function processIsRunning(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch (error) { - return (error as NodeJS.ErrnoException).code !== "ESRCH"; - } -} - -function tryAcquireLock(lockPath: string, filename: string): boolean { - try { - fs.writeFileSync( - lockPath, - `${JSON.stringify({ - filename, - pid: process.pid, - startedAt: new Date().toISOString(), - startedAtMs: Date.now(), - })}\n`, - { flag: "wx", mode: 0o600 }, - ); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; - return false; - } -} - -function sameLockIdentity(left: LockSnapshot, right: LockSnapshot | null): boolean { - return ( - right !== null && - left.dev === right.dev && - left.ino === right.ino && - left.mtimeMs === right.mtimeMs && - left.size === right.size && - left.contents === right.contents - ); -} - -function staleLockClaimPath(lockPath: string, snapshot: LockSnapshot): string { - const identity = crypto - .createHash("sha256") - .update(String(snapshot.dev)) - .update("\0") - .update(String(snapshot.ino)) - .update("\0") - .update(String(snapshot.mtimeMs)) - .update("\0") - .update(String(snapshot.size)) - .update("\0") - .update(snapshot.contents) - .digest("hex") - .slice(0, 24); - return `${lockPath}.reclaim-${identity}`; -} - -function reclaimStaleLock(lockPath: string): boolean { - if (cacheLockStaleMs <= 0) return false; - const snapshot = readLockSnapshot(lockPath); - if (snapshot === null || Date.now() - snapshot.mtimeMs < cacheLockStaleMs) return false; - const { pid } = parseLockOwner(snapshot.contents); - if (pid !== undefined && processIsRunning(pid)) return false; - - // A hard link is an atomic claim on this exact lock inode. Contenders that - // observed the same stale lock derive the same claim path, so only one can - // unlink it. If the lock was replaced before linkSync, the identity check - // below rejects the replacement without removing it. - const claimPath = staleLockClaimPath(lockPath, snapshot); - try { - fs.linkSync(lockPath, claimPath); - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - if (code === "EEXIST" || code === "ENOENT") return false; - throw error; - } - - try { - const claimed = readLockSnapshot(claimPath); - if (!sameLockIdentity(snapshot, claimed) || claimed === null) return false; - let current: fs.Stats; - try { - current = fs.lstatSync(lockPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; - throw error; - } - if (current.dev !== claimed.dev || current.ino !== claimed.ino) return false; - fs.unlinkSync(lockPath); - stats.staleLocks += 1; - return true; - } finally { - fs.rmSync(claimPath, { force: true }); - } -} - -function transpileSource(source: string, filename: string): string { - const start = nowMs(); - const result = ts.transpileModule(source, { - compilerOptions, - fileName: filename, - reportDiagnostics: true, - }); - stats.transformMs += nowMs() - start; - stats.transforms += 1; - const errors = result.diagnostics?.filter( - (diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error, - ); - if (errors && errors.length > 0) { - throw new Error( - errors - .map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")) - .join("\n"), - ); - } - return result.outputText; -} - -function waitForCache(cachePath: string): string | null { - if (cacheWaitMs <= 0) return null; - const start = nowMs(); - const deadline = start + cacheWaitMs; - stats.lockWaits += 1; - - while (nowMs() < deadline) { - const output = readCachedOutput(cachePath); - if (output !== null) { - stats.waitMs += nowMs() - start; - return output; - } - sleepSync(Math.min(cachePollMs, Math.max(0, deadline - nowMs()))); - } - stats.waitMs += nowMs() - start; - return null; -} - -function compileWithCache(filename: string, source: string, cachePath: string): string { - const cached = readCachedOutput(cachePath); - if (cached !== null) { - stats.cacheHits += 1; - return cached; - } - stats.cacheMisses += 1; - - const lockPath = `${cachePath}.lock`; - let ownsLock = tryAcquireLock(lockPath, filename); - if (!ownsLock && reclaimStaleLock(lockPath)) { - const cachedAfterReclaim = readCachedOutput(cachePath); - if (cachedAfterReclaim !== null) { - stats.cacheHits += 1; - return cachedAfterReclaim; - } - ownsLock = tryAcquireLock(lockPath, filename); - } - - if (!ownsLock) { - const waited = waitForCache(cachePath); - if (waited !== null) { - stats.cacheHits += 1; - return waited; - } - stats.duplicateFallbacks += 1; - return transpileSource(source, filename); - } - - try { - const outputText = transpileSource(source, filename); - try { - writeAtomic(cachePath, outputText); - } catch { - // The compiled output is still valid when best-effort cache persistence fails. - } - return outputText; - } finally { - fs.rmSync(lockPath, { force: true }); - } -} - -if (statsPath) { - process.once("exit", () => { - if (stats.files === 0) return; - let descriptor: number | null = null; - try { - const row = { - ...stats, - cacheDir, - cachePollMs, - label: process.env.NEMOCLAW_SOURCE_REQUIRE_STATS_LABEL ?? null, - pid: process.pid, - rssMb: Math.round((process.memoryUsage().rss / 1024 / 1024) * 10) / 10, - }; - descriptor = openSourceRequireStats(statsPath); - if (descriptor !== null) fs.appendFileSync(descriptor, `${JSON.stringify(row)}\n`); - } catch { - // Diagnostics are best-effort and must not fail an otherwise successful test process. - } finally { - if (descriptor !== null) fs.closeSync(descriptor); - } - }); -} const resolveFilename = moduleRuntime._resolveFilename; moduleRuntime._resolveFilename = function resolveSourceFilename(request, parent, isMain, options) { @@ -450,14 +67,17 @@ moduleRuntime._resolveFilename = function resolveSourceFilename(request, parent, }; moduleRuntime._extensions[".ts"] = (module, filename) => { - const compileStart = nowMs(); - stats.files += 1; - const source = fs.readFileSync(filename, "utf8"); - const cachePath = sourceRequireCachePath({ compilerOptions, filename, repoRoot, source }); - const outputText = compileWithCache(filename, source, cachePath); - stats.compileMs += nowMs() - compileStart; - module._compile(outputText, filename); + module._compile(compileSourceRequire(filename), filename); }; moduleRuntime._extensions[".cts"] = moduleRuntime._extensions[".ts"]; moduleRuntime._extensions[".mts"] = moduleRuntime._extensions[".ts"]; + +const loadJavaScript = moduleRuntime._extensions[".js"]; +moduleRuntime._extensions[".js"] = (module, filename) => { + if (!shouldInstrumentSource(filename)) return loadJavaScript(module, filename); + const source = fs.readFileSync(filename, "utf8"); + const instrumented = instrumentSourceForCoverage(source, filename); + if (instrumented) module._compile(instrumented.code, filename); + else loadJavaScript(module, filename); +}; diff --git a/test/helpers/source-coverage-plugin.ts b/test/helpers/source-coverage-plugin.ts new file mode 100644 index 00000000000..d7875fc8a23 --- /dev/null +++ b/test/helpers/source-coverage-plugin.ts @@ -0,0 +1,26 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import type { Plugin } from "vite"; + +export function sourceCoveragePlugin(): Plugin { + return { + name: "nemoclaw-source-coverage", + enforce: "pre", + config(config) { + // Blob replay installs placeholder transforms for recorded modules. + // Report merging executes no tests and can load the provider natively. + const merging = + config.test?.mergeReports || + process.argv.some((argument) => + /^(?:--mergeReports|--merge-reports)(?:=|$)/.test(argument), + ); + if (merging) return { test: { experimental: { viteModuleRunner: false } } }; + }, + transform(source, id) { + // The provider publishes the collector shared with the native loader. + const state = Reflect.get(globalThis, Symbol.for("nemoclaw.source-coverage.state")); + return state?.instrument(source, id); + }, + }; +} diff --git a/test/helpers/source-coverage-provider.mts b/test/helpers/source-coverage-provider.mts new file mode 100644 index 00000000000..6009cce3f26 --- /dev/null +++ b/test/helpers/source-coverage-provider.mts @@ -0,0 +1,18 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { COVERAGE_KEY, disableSourceCoverage, resetSourceCoverage } from "./source-coverage.cts"; + +export default { + startCoverage: resetSourceCoverage, + takeCoverage() { + const coverage = structuredClone(globalThis[COVERAGE_KEY] ?? {}); + resetSourceCoverage(); + return coverage; + }, + stopCoverage: disableSourceCoverage, + async getProvider() { + const { SourceCoverageProvider } = await import("./source-coverage-report.mts"); + return new SourceCoverageProvider(); + }, +}; diff --git a/test/helpers/source-coverage-report.mts b/test/helpers/source-coverage-report.mts new file mode 100644 index 00000000000..9e207e665c4 --- /dev/null +++ b/test/helpers/source-coverage-report.mts @@ -0,0 +1,120 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import fs from "node:fs"; +import { V8CoverageProvider } from "@vitest/coverage-v8/dist/provider.js"; +import { + OPTIONS_ENV, + disableSourceCoverage, + enableSourceCoverage, + instrumentSource, + instrumentSourceForCoverage, +} from "./source-coverage.cts"; + +// Reuse Vitest's file selection, reporters, security floors and shard transport. +// Only collection changes: original-source counters replace generated V8 ranges. +export class SourceCoverageProvider extends V8CoverageProvider { + name = "custom"; + + initialize(ctx) { + super.initialize(ctx); + this.previousOptions = process.env[OPTIONS_ENV]; + this.configureSourceCoverage(); + ctx.onClose(() => { + if (process.env[OPTIONS_ENV] !== this.serializedOptions) return; + disableSourceCoverage(); + if (this.previousOptions === undefined) delete process.env[OPTIONS_ENV]; + else process.env[OPTIONS_ENV] = this.previousOptions; + }); + } + + configureSourceCoverage() { + this.sourceOptions = { + roots: this.roots, + include: this.options.include, + exclude: this.options.exclude, + allowExternal: this.options.allowExternal, + ignoreClassMethods: this.options.ignoreClassMethods, + changedFiles: this.changedFiles, + }; + this.serializedOptions = JSON.stringify(this.sourceOptions); + process.env[OPTIONS_ENV] = this.serializedOptions; + enableSourceCoverage(this.sourceOptions, () => Boolean(this.ctx.coverageProvider)); + } + + async onTestRunStart() { + await super.onTestRunStart(); + this.configureSourceCoverage(); + const { compileSourceRequire } = await import("./source-require-compiler.ts"); + // Prepare transforms before per-test timers start. Compilation does not + // evaluate application modules or initialize their coverage counters. + for (const filename of await this.getUntestedFiles([])) { + const source = fs.readFileSync(filename, "utf8"); + instrumentSourceForCoverage(source, filename); + if (/\.[cm]?tsx?$/.test(filename)) compileSourceRequire(filename, source); + } + } + + requiresTransform(id) { + return this.isIncluded(id); + } + + onFileTransform() { + // The pre-transform plugin instruments original source. This hook tells + // Vitest that enabling coverage changes its persistent transform cache. + return null; + } + + onEnabled() { + this.configureSourceCoverage(); + for (const project of this.ctx.projects) { + for (const environment of Object.values(project.vite.environments)) { + environment.moduleGraph.invalidateAll(); + } + } + } + + mergeSourceCoverage(map, incoming) { + for (const [filename, coverage] of Object.entries(incoming)) { + if (!this.isIncluded(filename)) continue; + const identity = (value) => { + if (!value.statementMap || !value.fnMap || !value.branchMap) { + throw new Error(`Missing original-source coverage maps: ${filename}`); + } + return JSON.stringify([value.statementMap, value.fnMap, value.branchMap]); + }; + const incomingIdentity = identity(coverage); + if (map.files().includes(filename)) { + const previous = map.fileCoverageFor(filename).toJSON(); + if (identity(previous) !== incomingIdentity) { + throw new Error(`Conflicting original-source coverage maps: ${filename}`); + } + } + map.addFileCoverage(coverage); + } + } + + async generateCoverage({ allTestsRun }) { + const map = this.createCoverageMap(); + await this.readCoverageFiles({ + onFileRead: (coverage) => this.mergeSourceCoverage(map, coverage), + onFinished() {}, + onDebug: Object.assign(() => {}, { enabled: false }), + }); + if (this.options.include != null && (allTestsRun || !this.options.cleanOnRerun)) { + for (const filename of await this.getUntestedFiles(map.files())) { + const source = fs.readFileSync(filename, "utf8"); + const { coverage } = instrumentSource(source, filename, this.sourceOptions); + map.addFileCoverage(coverage); + } + } + map.filter((filename) => fs.existsSync(filename) && this.isIncluded(filename)); + return map; + } + + async mergeReports(coverageMaps) { + const map = this.createCoverageMap(); + for (const coverage of coverageMaps) this.mergeSourceCoverage(map, coverage); + await this.generateReports(map, true); + } +} diff --git a/test/helpers/source-coverage.cts b/test/helpers/source-coverage.cts new file mode 100644 index 00000000000..788ef4a7fdc --- /dev/null +++ b/test/helpers/source-coverage.cts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +const path = require("node:path"); +const { fileURLToPath } = require("node:url"); + +const OPTIONS_ENV = "NEMOCLAW_SOURCE_COVERAGE_OPTIONS"; +const COVERAGE_KEY = "__nemoclawSourceCoverage__"; +const STATE_KEY = Symbol.for("nemoclaw.source-coverage.state"); + +function enableSourceCoverage(options, isEnabled = () => true) { + // Compile once before enabling the hook that examines every native import. + const picomatch = require("picomatch"); + const matcher = picomatch(options.include ?? "**", { + contains: true, + dot: true, + ignore: options.exclude, + }); + globalThis[STATE_KEY] = { + options, + cache: new Map(), + matcher, + isEnabled, + instrument: instrumentSourceForCoverage, + }; +} + +function disableSourceCoverage() { + delete globalThis[STATE_KEY]; +} + +function resetSourceCoverage() { + const serialized = process.env[OPTIONS_ENV]; + if (!serialized) throw new Error("Source coverage options were not initialized"); + enableSourceCoverage(JSON.parse(serialized)); + // Keep objects referenced by cached modules alive across non-isolated runs. + for (const file of Object.values(globalThis[COVERAGE_KEY] ?? {})) { + for (const metric of ["s", "f"]) { + for (const key of Object.keys(file[metric])) file[metric][key] = 0; + } + for (const key of Object.keys(file.b)) file.b[key].fill(0); + } +} + +function canonicalFilename(id) { + const filename = id.startsWith("file:") ? fileURLToPath(id) : id.split("?")[0]; + return path.resolve(filename).replaceAll("\\", "/"); +} + +function isIncluded(filename, options, matcher) { + if ( + options.allowExternal === false && + !options.roots.some((root) => { + const relative = path.relative(root, filename); + return ( + !path.isAbsolute(relative) && relative !== ".." && !relative.startsWith(`..${path.sep}`) + ); + }) + ) + return false; + return matcher(filename) && (!options.changedFiles || options.changedFiles.includes(filename)); +} + +function guardSerializedCounters(program, types) { + const factory = program.node.body[0]; + if (factory?.loc || !types.isFunctionDeclaration(factory)) return; + const name = factory.id.name; + if (!name.startsWith("cov_")) return; + program.traverse({ + UpdateExpression(counter) { + if (counter.node.loc || !counter.getFunctionParent()) return; + let root = counter.node.argument; + while (types.isMemberExpression(root)) root = root.object; + if (!types.isCallExpression(root) || !types.isIdentifier(root.callee, { name })) return; + // A serialized function can run outside the instrumented module. Keep + // its behavior while counting every execution in the collecting process. + counter.replaceWith( + types.logicalExpression( + "&&", + types.binaryExpression( + "===", + types.unaryExpression("typeof", types.identifier(name)), + types.stringLiteral("function"), + ), + counter.node, + ), + ); + counter.skip(); + }, + }); +} + +function instrumentSource(source, filename, options) { + const { transformSync } = require("@babel/core"); + const { programVisitor, readInitialCoverage } = require("istanbul-lib-instrument"); + let coverage; + const result = transformSync(source, { + filename, + configFile: false, + babelrc: false, + ast: true, + sourceMaps: true, + compact: false, + comments: true, + parserOpts: { + sourceType: "module", + plugins: [ + "typescript", + ...(filename.endsWith("x") ? ["jsx"] : []), + ["importAttributes", { deprecatedAssertSyntax: true }], + ], + }, + plugins: [ + ({ types }) => { + const visitor = programVisitor(types, filename, { + coverageVariable: COVERAGE_KEY, + coverageGlobalScope: "globalThis", + coverageGlobalScopeFunc: false, + ignoreClassMethods: options.ignoreClassMethods, + }); + return { + visitor: { + Program: { + enter: visitor.enter, + exit(program) { + coverage = visitor.exit(program)?.fileCoverage; + if (coverage) guardSerializedCounters(program, types); + }, + }, + }, + }; + }, + ], + }); + if (!coverage) return { code: source, coverage: readInitialCoverage(result.ast)?.coverageData }; + return { code: result.code, map: result.map, coverage }; +} + +function shouldInstrumentSource(id) { + const state = globalThis[STATE_KEY]; + return Boolean( + state && + state.isEnabled() && + /\.[cm]?[jt]sx?(?:\?|$)/.test(id) && + isIncluded(canonicalFilename(id), state.options, state.matcher), + ); +} + +function instrumentSourceForCoverage(source, id) { + if (!shouldInstrumentSource(id)) return undefined; + const state = globalThis[STATE_KEY]; + const filename = canonicalFilename(id); + const cached = state.cache.get(filename); + if (cached?.source === source) return cached.result; + const result = instrumentSource(source, filename, state.options); + state.cache.set(filename, { source, result }); + return result; +} + +function sourceCoverageCacheIdentity(id) { + if (!shouldInstrumentSource(id)) return ""; + const state = globalThis[STATE_KEY]; + if (!state.fingerprint) { + const fs = require("node:fs"); + const lockfile = path.resolve(__dirname, "../../package-lock.json"); + const dependencies = fs.existsSync(lockfile) + ? fs.readFileSync(lockfile) + : JSON.stringify([ + require("@babel/core/package.json"), + require("istanbul-lib-instrument/package.json"), + ]); + state.fingerprint = require("node:crypto") + .createHash("sha256") + .update(fs.readFileSync(__filename)) + .update(dependencies) + .update(JSON.stringify(state.options.ignoreClassMethods ?? [])) + .digest("hex"); + } + return state.fingerprint; +} + +module.exports = { + OPTIONS_ENV, + COVERAGE_KEY, + enableSourceCoverage, + disableSourceCoverage, + resetSourceCoverage, + instrumentSource, + shouldInstrumentSource, + instrumentSourceForCoverage, + sourceCoverageCacheIdentity, +}; diff --git a/test/helpers/source-require-compiler.ts b/test/helpers/source-require-compiler.ts new file mode 100644 index 00000000000..c1225b264d8 --- /dev/null +++ b/test/helpers/source-require-compiler.ts @@ -0,0 +1,415 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { performance } from "node:perf_hooks"; +import ts from "typescript"; +import { instrumentSourceForCoverage, sourceCoverageCacheIdentity } from "./source-coverage.cts"; + +import { + loadSourceRequireCompilerOptions, + sourceRequireCacheDir, + sourceRequireCachePath, +} from "./source-require-cache"; + +type LockSnapshot = { + contents: string; + dev: number; + ino: number; + mtimeMs: number; + size: number; +}; + +const repoRoot = path.resolve(__dirname, "../.."); +const compilerOptions = loadSourceRequireCompilerOptions(repoRoot); +// Keep the cross-process transpilation cache in this checkout's dependency +// tree. A shared, predictable directory under the OS temp root could be +// replaced by another local user before a test process reads from it. +const cacheDir = sourceRequireCacheDir(repoRoot); +fs.mkdirSync(cacheDir, { recursive: true }); +const cacheWaitMs = envInt("NEMOCLAW_SOURCE_REQUIRE_CACHE_WAIT_MS", 5_000); +const cachePollMs = Math.max(1, envInt("NEMOCLAW_SOURCE_REQUIRE_CACHE_POLL_MS", 25)); +const cacheLockStaleMs = envInt("NEMOCLAW_SOURCE_REQUIRE_CACHE_LOCK_STALE_MS", 30_000); +const statsPath = sourceRequireStatsPath(process.env.NEMOCLAW_SOURCE_REQUIRE_STATS); + +const stats = { + cacheHits: 0, + cacheMisses: 0, + compileMs: 0, + duplicateFallbacks: 0, + files: 0, + lockWaits: 0, + readCacheMs: 0, + staleLocks: 0, + transforms: 0, + transformMs: 0, + waitMs: 0, +}; + +function envInt(name: string, fallback: number): number { + const raw = process.env[name]; + if (raw === undefined) return fallback; + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +} + +function isWithin(root: string, candidate: string): boolean { + const relative = path.relative(root, candidate); + return ( + relative === "" || + (!path.isAbsolute(relative) && !relative.startsWith(`..${path.sep}`) && relative !== "..") + ); +} + +function sourceRequireStatsPath(configuredPath: string | undefined): string | undefined { + if (!configuredPath) return undefined; + const resolved = path.resolve(configuredPath); + return [repoRoot, os.tmpdir()].some((root) => isWithin(path.resolve(root), resolved)) + ? resolved + : undefined; +} + +function openSourceRequireStats(statsPath: string): number | null { + const parent = path.dirname(statsPath); + let canonicalParent: string; + try { + canonicalParent = fs.realpathSync.native(parent); + } catch { + return null; + } + const canonicalPath = path.join(canonicalParent, path.basename(statsPath)); + const withinAllowedRoot = [repoRoot, os.tmpdir()].some((root) => { + try { + return isWithin(fs.realpathSync.native(root), canonicalPath); + } catch { + return false; + } + }); + if (!withinAllowedRoot) return null; + + let descriptor: number; + try { + descriptor = fs.openSync( + statsPath, + fs.constants.O_APPEND | + fs.constants.O_CREAT | + fs.constants.O_WRONLY | + fs.constants.O_NOFOLLOW, + 0o600, + ); + } catch { + return null; + } + const stat = fs.fstatSync(descriptor); + if (!stat.isFile() || stat.nlink !== 1) { + fs.closeSync(descriptor); + return null; + } + return descriptor; +} + +function nowMs(): number { + return performance.now(); +} + +const sleepBuffer = new SharedArrayBuffer(4); +const sleepArray = new Int32Array(sleepBuffer); + +function sleepSync(ms: number): void { + if (ms <= 0) return; + Atomics.wait(sleepArray, 0, 0, ms); +} + +function readCachedOutput(cachePath: string): string | null { + const start = nowMs(); + try { + const output = fs.readFileSync(cachePath, "utf8"); + stats.readCacheMs += nowMs() - start; + return output; + } catch (error) { + stats.readCacheMs += nowMs() - start; + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + return null; + } +} + +function writeAtomic(cachePath: string, outputText: string): void { + const temporaryPath = `${cachePath}.${process.pid}.${crypto.randomUUID()}`; + fs.writeFileSync(temporaryPath, outputText, { flag: "wx", mode: 0o600 }); + try { + fs.renameSync(temporaryPath, cachePath); + } catch (error) { + fs.rmSync(temporaryPath, { force: true }); + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + } +} + +function readLockSnapshot(lockPath: string): LockSnapshot | null { + let descriptor: number; + try { + descriptor = fs.openSync(lockPath, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "ELOOP") return null; + throw error; + } + + try { + const before = fs.fstatSync(descriptor); + const contents = fs.readFileSync(descriptor, "utf8"); + const after = fs.fstatSync(descriptor); + if ( + before.dev !== after.dev || + before.ino !== after.ino || + before.mtimeMs !== after.mtimeMs || + before.size !== after.size + ) { + return null; + } + return { + contents, + dev: after.dev, + ino: after.ino, + mtimeMs: after.mtimeMs, + size: after.size, + }; + } finally { + fs.closeSync(descriptor); + } +} + +function parseLockOwner(contents: string): { pid?: number } { + try { + const parsed = JSON.parse(contents); + return typeof parsed?.pid === "number" && Number.isInteger(parsed.pid) && parsed.pid > 0 + ? { pid: parsed.pid } + : {}; + } catch { + return {}; + } +} + +function processIsRunning(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== "ESRCH"; + } +} + +function tryAcquireLock(lockPath: string, filename: string): boolean { + try { + fs.writeFileSync( + lockPath, + `${JSON.stringify({ + filename, + pid: process.pid, + startedAt: new Date().toISOString(), + startedAtMs: Date.now(), + })}\n`, + { flag: "wx", mode: 0o600 }, + ); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + return false; + } +} + +function sameLockIdentity(left: LockSnapshot, right: LockSnapshot | null): boolean { + return ( + right !== null && + left.dev === right.dev && + left.ino === right.ino && + left.mtimeMs === right.mtimeMs && + left.size === right.size && + left.contents === right.contents + ); +} + +function staleLockClaimPath(lockPath: string, snapshot: LockSnapshot): string { + const identity = crypto + .createHash("sha256") + .update(String(snapshot.dev)) + .update("\0") + .update(String(snapshot.ino)) + .update("\0") + .update(String(snapshot.mtimeMs)) + .update("\0") + .update(String(snapshot.size)) + .update("\0") + .update(snapshot.contents) + .digest("hex") + .slice(0, 24); + return `${lockPath}.reclaim-${identity}`; +} + +function reclaimStaleLock(lockPath: string): boolean { + if (cacheLockStaleMs <= 0) return false; + const snapshot = readLockSnapshot(lockPath); + if (snapshot === null || Date.now() - snapshot.mtimeMs < cacheLockStaleMs) return false; + const { pid } = parseLockOwner(snapshot.contents); + if (pid !== undefined && processIsRunning(pid)) return false; + + // A hard link is an atomic claim on this exact lock inode. Contenders that + // observed the same stale lock derive the same claim path, so only one can + // unlink it. If the lock was replaced before linkSync, the identity check + // below rejects the replacement without removing it. + const claimPath = staleLockClaimPath(lockPath, snapshot); + try { + fs.linkSync(lockPath, claimPath); + } catch (error) { + const code = (error as NodeJS.ErrnoException).code; + if (code === "EEXIST" || code === "ENOENT") return false; + throw error; + } + + try { + const claimed = readLockSnapshot(claimPath); + if (!sameLockIdentity(snapshot, claimed) || claimed === null) return false; + let current: fs.Stats; + try { + current = fs.lstatSync(lockPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } + if (current.dev !== claimed.dev || current.ino !== claimed.ino) return false; + fs.unlinkSync(lockPath); + stats.staleLocks += 1; + return true; + } finally { + fs.rmSync(claimPath, { force: true }); + } +} + +function transpileSource(source: string, filename: string): string { + const start = nowMs(); + const result = ts.transpileModule(source, { + compilerOptions, + fileName: filename, + reportDiagnostics: true, + }); + stats.transformMs += nowMs() - start; + stats.transforms += 1; + const errors = result.diagnostics?.filter( + (diagnostic) => diagnostic.category === ts.DiagnosticCategory.Error, + ); + if (errors && errors.length > 0) { + throw new Error( + errors + .map((diagnostic) => ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")) + .join("\n"), + ); + } + return result.outputText; +} + +function waitForCache(cachePath: string): string | null { + if (cacheWaitMs <= 0) return null; + const start = nowMs(); + const deadline = start + cacheWaitMs; + stats.lockWaits += 1; + + while (nowMs() < deadline) { + const output = readCachedOutput(cachePath); + if (output !== null) { + stats.waitMs += nowMs() - start; + return output; + } + sleepSync(Math.min(cachePollMs, Math.max(0, deadline - nowMs()))); + } + stats.waitMs += nowMs() - start; + return null; +} + +function compileWithCache(filename: string, source: () => string, cachePath: string): string { + const cached = readCachedOutput(cachePath); + if (cached !== null) { + stats.cacheHits += 1; + return cached; + } + stats.cacheMisses += 1; + + const lockPath = `${cachePath}.lock`; + let ownsLock = tryAcquireLock(lockPath, filename); + if (!ownsLock && reclaimStaleLock(lockPath)) { + const cachedAfterReclaim = readCachedOutput(cachePath); + if (cachedAfterReclaim !== null) { + stats.cacheHits += 1; + return cachedAfterReclaim; + } + ownsLock = tryAcquireLock(lockPath, filename); + } + + if (!ownsLock) { + const waited = waitForCache(cachePath); + if (waited !== null) { + stats.cacheHits += 1; + return waited; + } + stats.duplicateFallbacks += 1; + return transpileSource(source(), filename); + } + + try { + const outputText = transpileSource(source(), filename); + try { + writeAtomic(cachePath, outputText); + } catch { + // The compiled output is still valid when best-effort cache persistence fails. + } + return outputText; + } finally { + fs.rmSync(lockPath, { force: true }); + } +} + +if (statsPath) { + process.once("exit", () => { + if (stats.files === 0) return; + let descriptor: number | null = null; + try { + const row = { + ...stats, + cacheDir, + cachePollMs, + label: process.env.NEMOCLAW_SOURCE_REQUIRE_STATS_LABEL ?? null, + pid: process.pid, + rssMb: Math.round((process.memoryUsage().rss / 1024 / 1024) * 10) / 10, + }; + descriptor = openSourceRequireStats(statsPath); + if (descriptor !== null) fs.appendFileSync(descriptor, `${JSON.stringify(row)}\n`); + } catch { + // Diagnostics are best-effort and must not fail an otherwise successful test process. + } finally { + if (descriptor !== null) fs.closeSync(descriptor); + } + }); +} + +export function compileSourceRequire( + filename: string, + originalSource = fs.readFileSync(filename, "utf8"), +): string { + const compileStart = nowMs(); + stats.files += 1; + const coverageIdentity = sourceCoverageCacheIdentity(filename); + const cachePath = sourceRequireCachePath({ + compilerOptions, + filename, + repoRoot, + source: coverageIdentity ? `${originalSource}\0${coverageIdentity}` : originalSource, + }); + const outputText = compileWithCache( + filename, + () => instrumentSourceForCoverage(originalSource, filename)?.code ?? originalSource, + cachePath, + ); + stats.compileMs += nowMs() - compileStart; + return outputText; +} diff --git a/test/repository/source-coverage.test.ts b/test/repository/source-coverage.test.ts new file mode 100644 index 00000000000..d4ca358b787 --- /dev/null +++ b/test/repository/source-coverage.test.ts @@ -0,0 +1,317 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it } from "vitest"; +import { + nodeOptionsWithoutSourceLoader, + SOURCE_REQUIRE_HOOK, +} from "../helpers/source-loader-options"; +import { testTimeoutOptions } from "../helpers/timeouts"; + +const repositoryRoot = path.resolve(import.meta.dirname, "../.."); +const roots: string[] = []; +type Coverage = { + fnMap: Record; + f: Record; + s: Record; + b: Record; +}; + +afterEach(() => { + for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); +}); + +function createFixture(mode: string, floor?: number) { + const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-source-coverage-")); + roots.push(root); + fs.mkdirSync(path.join(root, "src")); + fs.symlinkSync(path.join(repositoryRoot, "node_modules"), path.join(root, "node_modules"), "dir"); + fs.writeFileSync(path.join(root, "package.json"), '{"type":"module"}'); + fs.writeFileSync( + path.join(root, "src/probe.ts"), + ` +export function nativeBranch(flag: boolean): string { + if (flag) return "native-true"; + return "native-false"; +} +export function ssrBranch(flag: boolean): string { + if (flag) return "ssr-true"; + return "ssr-false"; +} +export function neverCalled(): string { return "never"; } +`, + ); + fs.writeFileSync( + path.join(root, "src/unloaded.ts"), + 'throw new Error("Unloaded source was executed during preparation"); export function unloaded() { return 7; }', + ); + fs.writeFileSync(path.join(root, "src/ignored.ts"), "export function ignored() { return 9; }"); + fs.writeFileSync(path.join(root, "src/native.cjs"), "exports.nativeJs = () => 42;"); + fs.writeFileSync( + path.join(root, "probe.test.ts"), + ` +import {createRequire} from "node:module"; +import {runInNewContext} from "node:vm"; +import {it, expect} from "vitest"; +import * as ssr from "./src/probe"; +import {ignored} from "./src/ignored"; +const native = createRequire(import.meta.url)("./src/probe.ts"); +const nativeJs = createRequire(import.meta.url)("./src/native.cjs"); +it("executes only the selected loader paths", () => { + if (${JSON.stringify(mode)} !== "omit-native") expect(native.nativeBranch(true)).toBe("native-true"); + if (${JSON.stringify(mode)} !== "omit-ssr") expect(ssr.ssrBranch(false)).toBe("ssr-false"); + expect(runInNewContext("(" + native.nativeBranch.toString() + ")(false)")).toBe("native-false"); + expect(runInNewContext("(" + ssr.ssrBranch.toString() + ")(true)")).toBe("ssr-true"); + expect(nativeJs.nativeJs()).toBe(42); + expect(ignored()).toBe(9); +}); +`, + ); + fs.writeFileSync( + path.join(root, "vitest.config.ts"), + ` +import {defineConfig} from "vitest/config"; +import {sourceCoveragePlugin} from ${JSON.stringify(path.join(repositoryRoot, "test/helpers/source-coverage-plugin.ts"))}; +export default defineConfig({ + root: ${JSON.stringify(root)}, + plugins: [sourceCoveragePlugin()], + test: { + include: ["*.test.ts"], + setupFiles: [${JSON.stringify(SOURCE_REQUIRE_HOOK)}], + coverage: { + provider: "custom", + customProviderModule: ${JSON.stringify(path.join(repositoryRoot, "test/helpers/source-coverage-provider.mts"))}, + include: ["src/**/*.ts", "src/**/*.cjs"], + exclude: ["src/ignored.ts"], + reporter: ["json"], + reportsDirectory: "coverage", + thresholds: ${JSON.stringify(floor ? { functions: floor } : {})}, + }, + }, +}); +`, + ); + return root; +} + +function runFixture(root: string, args = ["run", "--coverage"]) { + return spawnSync( + process.execPath, + [path.join(repositoryRoot, "node_modules/vitest/vitest.mjs"), ...args, "--maxWorkers=1"], + { + cwd: root, + env: { + ...process.env, + NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), + }, + encoding: "utf8", + timeout: 30_000, + }, + ); +} + +function functions(coverage: Coverage) { + return Object.fromEntries( + Object.entries(coverage.fnMap).map(([id, entry]) => [entry.name, coverage.f[id]]), + ); +} + +function createShardFixture() { + const root = createFixture("both"); + fs.copyFileSync(path.join(root, "probe.test.ts"), path.join(root, "second.test.ts")); + return root; +} + +function expectShardSuccess(root: string, shard: string) { + const result = runFixture(root, ["run", "--coverage", `--shard=${shard}`, "--reporter=blob"]); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); +} + +describe("original-source coverage across loaders", () => { + it( + "collects native counters from a warm disk cache without loading the instrumenter", + testTimeoutOptions(35_000), + () => { + const root = createFixture("both"); + const warmup = runFixture(root); + expect(warmup.status, `${warmup.stdout}\n${warmup.stderr}`).toBe(0); + const filename = path.join(root, "src/probe.ts"); + const script = ` +require(${JSON.stringify(SOURCE_REQUIRE_HOOK)}); +const coverage = require(${JSON.stringify(path.join(repositoryRoot, "test/helpers/source-coverage.cts"))}); +coverage.enableSourceCoverage({roots:[${JSON.stringify(root)}],include:[${JSON.stringify(filename)}],exclude:[],allowExternal:false}); +const Module = require("node:module"), load = Module._load; +Module._load = function(request, ...args) { + if (request === "@babel/core" || request === "istanbul-lib-instrument") throw new Error("Warm native cache attempted instrumentation"); + return load.call(this, request, ...args); +}; +const probe = require(${JSON.stringify(filename)}); +const value = probe.nativeBranch(true); +console.log(JSON.stringify({value,coverage:globalThis[coverage.COVERAGE_KEY][${JSON.stringify(filename)}]})); +`; + const result = spawnSync(process.execPath, ["-e", script], { + env: { + ...process.env, + NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), + }, + encoding: "utf8", + timeout: 10_000, + }); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const output = JSON.parse(result.stdout); + expect(output.value).toBe("native-true"); + expect(functions(output.coverage)).toEqual({ nativeBranch: 1, ssrBranch: 0, neverCalled: 0 }); + expect(Object.values(output.coverage.b)).toEqual([ + [1, 0], + [0, 0], + ]); + }, + ); + + it( + "collects accurate counters after coverage is enabled and reenabled", + testTimeoutOptions(35_000), + () => { + const root = createFixture("both"); + fs.writeFileSync( + path.join(root, "toggle.mjs"), + ` +import fs from "node:fs"; +import {createVitest} from "vitest/node"; +const ctx = await createVitest("test", { + root: ${JSON.stringify(root)}, watch: false, maxWorkers: 1, + coverage: {enabled: false}, + experimental: {fsModuleCache: true, fsModuleCachePath: "module-cache"}, +}); +try { + await ctx.start(); + await ctx.enableCoverage(); + await ctx.rerunTestSpecifications(await ctx.globTestSpecifications(), true); + fs.copyFileSync("coverage/coverage-final.json", "first.json"); + ctx.disableCoverage(); + await ctx.rerunTestSpecifications(await ctx.globTestSpecifications(), true); + await ctx.enableCoverage(); + await ctx.rerunTestSpecifications(await ctx.globTestSpecifications(), true); + fs.copyFileSync("coverage/coverage-final.json", "second.json"); +} finally { await ctx.close(); } +`, + ); + const result = spawnSync(process.execPath, [path.join(root, "toggle.mjs")], { + cwd: root, + env: { + ...process.env, + NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), + }, + encoding: "utf8", + timeout: 30_000, + }); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const first = JSON.parse(fs.readFileSync(path.join(root, "first.json"), "utf8")); + const second = JSON.parse(fs.readFileSync(path.join(root, "second.json"), "utf8")); + expect(functions(first[path.join(root, "src/probe.ts")])).toEqual({ + nativeBranch: 1, + ssrBranch: 1, + neverCalled: 0, + }); + expect(functions(second[path.join(root, "src/probe.ts")])).toEqual({ + nativeBranch: 1, + ssrBranch: 1, + neverCalled: 0, + }); + }, + ); + + it("counts cached native and SSR modules once per suite", testTimeoutOptions(35_000), () => { + const root = createFixture("both"); + fs.copyFileSync(path.join(root, "probe.test.ts"), path.join(root, "second.test.ts")); + const result = runFixture(root, ["run", "--coverage", "--no-isolate", "--no-file-parallelism"]); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const report = JSON.parse( + fs.readFileSync(path.join(root, "coverage/coverage-final.json"), "utf8"), + ); + expect(functions(report[path.join(root, "src/probe.ts")])).toEqual({ + nativeBranch: 2, + ssrBranch: 2, + neverCalled: 0, + }); + expect(Object.values(report[path.join(root, "src/native.cjs")].f)).toEqual([2]); + }); + + it( + "merges matching shard reports with one entry per source construct", + testTimeoutOptions(35_000), + () => { + const root = createShardFixture(); + expectShardSuccess(root, "1/2"); + expectShardSuccess(root, "2/2"); + const merged = runFixture(root, ["--merge-reports", "--coverage"]); + expect(merged.status, `${merged.stdout}\n${merged.stderr}`).toBe(0); + const report = JSON.parse( + fs.readFileSync(path.join(root, "coverage/coverage-final.json"), "utf8"), + ); + expect(functions(report[path.join(root, "src/probe.ts")])).toEqual({ + nativeBranch: 2, + ssrBranch: 2, + neverCalled: 0, + }); + expect(functions(report[path.join(root, "src/unloaded.ts")])).toEqual({ unloaded: 0 }); + expect(Object.values(report[path.join(root, "src/unloaded.ts")].s)).not.toContain(1); + }, + ); + + it("rejects conflicting source maps when merging shards", testTimeoutOptions(35_000), () => { + const root = createShardFixture(); + expectShardSuccess(root, "1/2"); + const probe = path.join(root, "src/probe.ts"); + fs.writeFileSync(probe, `// Changed source locations\n${fs.readFileSync(probe, "utf8")}`); + expectShardSuccess(root, "2/2"); + const merged = runFixture(root, ["--merge-reports", "--coverage"]); + expect(merged.status).toBe(1); + expect(`${merged.stdout}\n${merged.stderr}`).toContain( + "Conflicting original-source coverage maps", + ); + }); + + it.each(["both", "omit-native", "omit-ssr"])( + "counts %s execution without duplicate source entries", + testTimeoutOptions(35_000), + (mode) => { + const root = createFixture(mode); + const result = runFixture(root); + expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); + const report = JSON.parse( + fs.readFileSync(path.join(root, "coverage/coverage-final.json"), "utf8"), + ) as Record; + const probe = report[path.join(root, "src/probe.ts")]; + expect(Object.keys(probe.f)).toHaveLength(3); + expect(functions(probe)).toEqual({ + nativeBranch: mode === "omit-native" ? 0 : 1, + ssrBranch: mode === "omit-ssr" ? 0 : 1, + neverCalled: 0, + }); + expect(Object.values(probe.b)).toEqual([ + mode === "omit-native" ? [0, 0] : [1, 0], + mode === "omit-ssr" ? [0, 0] : [0, 1], + ]); + expect(functions(report[path.join(root, "src/unloaded.ts")])).toEqual({ unloaded: 0 }); + expect(Object.values(report[path.join(root, "src/unloaded.ts")].s)).not.toContain(1); + expect(Object.values(report[path.join(root, "src/native.cjs")].f)).toEqual([1]); + expect(report[path.join(root, "src/ignored.ts")]).toBeUndefined(); + }, + ); + + it( + "fails an unmet coverage floor after collecting both loaders", + testTimeoutOptions(35_000), + () => { + const root = createFixture("both", 90); + const result = runFixture(root); + expect(result.status).toBe(1); + expect(`${result.stdout}\n${result.stderr}`).toContain("Coverage for functions"); + }, + ); +}); diff --git a/test/repository/source-require-loader.test.ts b/test/repository/source-require-loader.test.ts index 685cf37723f..35f644be1c5 100644 --- a/test/repository/source-require-loader.test.ts +++ b/test/repository/source-require-loader.test.ts @@ -171,6 +171,14 @@ process.exitCode = sdk.gatewayPort({ kind: "named", gatewayName: "nemoclaw" }) = path.join(REPO_ROOT, "test/helpers/source-require-cache.ts"), path.join(root, "test/helpers/source-require-cache.ts"), ); + fs.copyFileSync( + path.join(REPO_ROOT, "test/helpers/source-require-compiler.ts"), + path.join(root, "test/helpers/source-require-compiler.ts"), + ); + fs.copyFileSync( + path.join(REPO_ROOT, "test/helpers/source-coverage.cts"), + path.join(root, "test/helpers/source-coverage.cts"), + ); fs.symlinkSync( path.join(REPO_ROOT, "node_modules/typescript"), path.join(root, "node_modules/typescript"), @@ -558,6 +566,7 @@ const nativeTypeScriptLoader = Module._extensions[".ts"]; const expected = new Set([ path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "register-source-require.ts"))}), path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "source-require-cache.ts"))}), + path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "source-require-compiler.ts"))}), ]); const compiled = []; const originalCompile = Module.prototype._compile; @@ -603,7 +612,7 @@ if (require.cache[typescriptPath] !== undefined) { require.cache[typescriptPath] === undefined || registeredTypeScriptLoader === nativeTypeScriptLoader || !rejectedUnexpected || - compiled.length !== 2 || + compiled.length !== 3 || compiled.some((entry) => !entry.sourceMapped) ) { console.error(JSON.stringify({ fixture, compiled, rejectedUnexpected, typescriptLoaded: require.cache[typescriptPath] !== undefined })); diff --git a/vitest.config.ts b/vitest.config.ts index ee953e299b7..bf290d30164 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -9,6 +9,7 @@ import { defineConfig, defineProject } from "vitest/config"; import pluginVitestProjectOptions from "./nemoclaw/vitest.project"; import { shouldRunLiveE2E } from "./test/e2e/fixtures/live-project-gate.ts"; import { CliCoverageSequencer } from "./test/helpers/cli-coverage-sequencer"; +import { sourceCoveragePlugin } from "./test/helpers/source-coverage-plugin"; import { resolveCliCoverageShardScheduling, resolveIntegrationProjectScheduling, @@ -95,6 +96,7 @@ const e2ePhaseCollectionAlias = ] : []; const typedSourceTransform = { + plugins: [sourceCoveragePlugin()], oxc: { include: /\.(?:[cm]?ts|[jt]sx)$/, }, @@ -112,7 +114,10 @@ const controlledNonLiveEnv = { // test/helpers/normalize-fixture-umask.ts (#6448). const fixtureUmaskSetup = "test/helpers/normalize-fixture-umask.ts"; const isolatedTestStateSetup = "test/helpers/isolate-test-state.ts"; -const pluginVitestProject = defineProject(pluginVitestProjectOptions); +const pluginVitestProject = defineProject({ + ...pluginVitestProjectOptions, + plugins: [sourceCoveragePlugin()], +}); // Pull-request jobs execute the base branch's trusted composite action, so an // action change in a PR cannot constrain that PR's own Vitest workers. Apply a // bounded cap from the validated shard environment instead; this is shared by the @@ -277,7 +282,8 @@ export default defineConfig({ }, ], coverage: { - provider: "v8", + provider: "custom", + customProviderModule: "./test/helpers/source-coverage-provider.mts", include: ["src/**/*.ts", "bin/**/*.js", "nemoclaw/src/**/*.ts", "nemoclaw/src/**/*.cts"], exclude: ["**/*.test.ts", "dist/**"], reporter: ["text-summary", "json-summary"], From 8066d53c7527f892c75f0e2327b0be6e24c66898 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Tue, 6 Oct 2026 16:56:47 -0700 Subject: [PATCH 2/5] fix(test): typecheck the native coverage bootstrap Signed-off-by: Prekshi Vyas --- package-lock.json | 58 +++++- package.json | 2 + test/helpers/onboard-script-mocks.cjs | 2 + test/helpers/register-source-require.ts | 5 +- test/helpers/source-coverage-plugin.ts | 11 +- test/helpers/source-coverage.cts | 172 +++++++++++++----- test/helpers/source-require-compiler.ts | 5 +- test/repository/source-require-loader.test.ts | 3 +- 8 files changed, 207 insertions(+), 51 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5f0f5907df9..34b3ecf0738 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,8 +8,8 @@ "name": "nemoclaw", "version": "0.1.0", "bundleDependencies": [ - "p-retry", - "@nvidia/openshell-sdk" + "@nvidia/openshell-sdk", + "p-retry" ], "hasInstallScript": true, "license": "Apache-2.0", @@ -40,7 +40,9 @@ "@commitlint/config-conventional": "^20.5.0", "@earendil-works/pi-coding-agent": "0.80.6", "@j178/prek": "^0.3.6", + "@types/babel__core": "7.20.5", "@types/node": "^25.5.2", + "@types/picomatch": "4.0.3", "@vitest/coverage-v8": "^4.1.0", "convert-source-map": "2.0.0", "eslint-plugin-sonarjs": "4.2.0", @@ -5258,6 +5260,51 @@ "tslib": "^2.4.0" } }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, "node_modules/@types/chai": { "version": "5.2.3", "resolved": "https://registry.npmjs.org/@types/chai/-/chai-5.2.3.tgz", @@ -5309,6 +5356,13 @@ "undici-types": "~7.18.0" } }, + "node_modules/@types/picomatch": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/@types/picomatch/-/picomatch-4.0.3.tgz", + "integrity": "sha512-iG0T6+nYJ9FAPmx9SsUlnwcq1ZVRuCXcVEvWnntoPlrOpwtSTKNDC9uVAxTsC3PUvJ+99n4RpAcNgBbHX3JSnQ==", + "dev": true, + "license": "MIT" + }, "node_modules/@types/retry": { "version": "0.12.0", "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", diff --git a/package.json b/package.json index d4220fa4289..0091a60c3ac 100644 --- a/package.json +++ b/package.json @@ -170,7 +170,9 @@ "@commitlint/config-conventional": "^20.5.0", "@earendil-works/pi-coding-agent": "0.80.6", "@j178/prek": "^0.3.6", + "@types/babel__core": "7.20.5", "@types/node": "^25.5.2", + "@types/picomatch": "4.0.3", "@vitest/coverage-v8": "^4.1.0", "convert-source-map": "2.0.0", "eslint-plugin-sonarjs": "4.2.0", diff --git a/test/helpers/onboard-script-mocks.cjs b/test/helpers/onboard-script-mocks.cjs index 95751ac27f5..dc33e9bec54 100644 --- a/test/helpers/onboard-script-mocks.cjs +++ b/test/helpers/onboard-script-mocks.cjs @@ -99,6 +99,7 @@ function registerSourceRequire() { path.resolve(sourceLoader), path.resolve(__dirname, "source-require-cache.ts"), path.resolve(__dirname, "source-require-compiler.ts"), + path.resolve(__dirname, "source-coverage.cts"), ]); const previousTypeScriptLoader = Module._extensions[".ts"]; @@ -125,6 +126,7 @@ function registerSourceRequire() { }); targetModule._compile(outputText, filename); }; + Module._extensions[".cts"] = Module._extensions[".ts"]; require(sourceLoader); } diff --git a/test/helpers/register-source-require.ts b/test/helpers/register-source-require.ts index 38ee364f961..2e2105af934 100644 --- a/test/helpers/register-source-require.ts +++ b/test/helpers/register-source-require.ts @@ -4,7 +4,10 @@ import fs from "node:fs"; import Module from "node:module"; import path from "node:path"; -import { instrumentSourceForCoverage, shouldInstrumentSource } from "./source-coverage.cts"; +const { + instrumentSourceForCoverage, + shouldInstrumentSource, +}: typeof import("./source-coverage.cts") = require("./source-coverage.cts"); import { compileSourceRequire } from "./source-require-compiler"; type CommonJsModule = NodeModule & { diff --git a/test/helpers/source-coverage-plugin.ts b/test/helpers/source-coverage-plugin.ts index d7875fc8a23..31cb929238d 100644 --- a/test/helpers/source-coverage-plugin.ts +++ b/test/helpers/source-coverage-plugin.ts @@ -11,11 +11,18 @@ export function sourceCoveragePlugin(): Plugin { // Blob replay installs placeholder transforms for recorded modules. // Report merging executes no tests and can load the provider natively. const merging = - config.test?.mergeReports || + Reflect.get(config.test ?? {}, "mergeReports") || process.argv.some((argument) => /^(?:--mergeReports|--merge-reports)(?:=|$)/.test(argument), ); - if (merging) return { test: { experimental: { viteModuleRunner: false } } }; + return { + test: { + // The shared CommonJS collector is typed, but Vite parses .cts as + // JavaScript. Node strips its types and preserves the native cache. + server: { deps: { external: [/[/\\]test[/\\]helpers[/\\]source-coverage\.cts$/] } }, + ...(merging ? { experimental: { viteModuleRunner: false } } : {}), + }, + }; }, transform(source, id) { // The provider publishes the collector shared with the native loader. diff --git a/test/helpers/source-coverage.cts b/test/helpers/source-coverage.cts index 788ef4a7fdc..7c2c06bc36f 100644 --- a/test/helpers/source-coverage.cts +++ b/test/helpers/source-coverage.cts @@ -4,11 +4,64 @@ const path = require("node:path"); const { fileURLToPath } = require("node:url"); -const OPTIONS_ENV = "NEMOCLAW_SOURCE_COVERAGE_OPTIONS"; -const COVERAGE_KEY = "__nemoclawSourceCoverage__"; +const OPTIONS_ENVImpl = "NEMOCLAW_SOURCE_COVERAGE_OPTIONS"; +const COVERAGE_KEYImpl = "__nemoclawSourceCoverage__"; const STATE_KEY = Symbol.for("nemoclaw.source-coverage.state"); -function enableSourceCoverage(options, isEnabled = () => true) { +type BabelTypes = typeof import("@babel/types"); +type CoverageOptions = { + roots: string[]; + include?: string | string[]; + exclude?: string[]; + allowExternal?: boolean; + changedFiles?: string[]; + ignoreClassMethods?: string[]; +}; +type FileCoverage = { + s: Record; + f: Record; + b: Record; + statementMap: Record; + fnMap: Record; + branchMap: Record; +}; +type InstrumentedSource = { code: string; map?: unknown; coverage?: FileCoverage }; +type CoverageState = { + options: CoverageOptions; + cache: Map; + matcher: (filename: string) => boolean; + isEnabled: () => boolean; + instrument: typeof instrumentSourceForCoverageImpl; + fingerprint?: string; +}; +type CounterPath = { + node: import("@babel/types").UpdateExpression; + getFunctionParent(): unknown; + replaceWith(node: import("@babel/types").Expression): void; + skip(): void; +}; +type ProgramPath = { + node: import("@babel/types").Program; + traverse(visitor: { UpdateExpression(counter: CounterPath): void }): void; +}; +const coverageGlobal = globalThis as typeof globalThis & { + [STATE_KEY]?: CoverageState; + [COVERAGE_KEYImpl]?: Record; +}; + +// Node loads this collector as native CommonJS in strip-only mode. These +// declarations expose its typed API without emitting unsupported export syntax. +export declare const OPTIONS_ENV: typeof OPTIONS_ENVImpl; +export declare const COVERAGE_KEY: typeof COVERAGE_KEYImpl; +export declare const enableSourceCoverage: typeof enableSourceCoverageImpl; +export declare const disableSourceCoverage: typeof disableSourceCoverageImpl; +export declare const resetSourceCoverage: typeof resetSourceCoverageImpl; +export declare const instrumentSource: typeof instrumentSourceImpl; +export declare const shouldInstrumentSource: typeof shouldInstrumentSourceImpl; +export declare const instrumentSourceForCoverage: typeof instrumentSourceForCoverageImpl; +export declare const sourceCoverageCacheIdentity: typeof sourceCoverageCacheIdentityImpl; + +function enableSourceCoverageImpl(options: CoverageOptions, isEnabled = () => true) { // Compile once before enabling the hook that examines every native import. const picomatch = require("picomatch"); const matcher = picomatch(options.include ?? "**", { @@ -16,38 +69,42 @@ function enableSourceCoverage(options, isEnabled = () => true) { dot: true, ignore: options.exclude, }); - globalThis[STATE_KEY] = { + coverageGlobal[STATE_KEY] = { options, cache: new Map(), matcher, isEnabled, - instrument: instrumentSourceForCoverage, + instrument: instrumentSourceForCoverageImpl, }; } -function disableSourceCoverage() { - delete globalThis[STATE_KEY]; +function disableSourceCoverageImpl() { + delete coverageGlobal[STATE_KEY]; } -function resetSourceCoverage() { - const serialized = process.env[OPTIONS_ENV]; +function resetSourceCoverageImpl() { + const serialized = process.env[OPTIONS_ENVImpl]; if (!serialized) throw new Error("Source coverage options were not initialized"); - enableSourceCoverage(JSON.parse(serialized)); + enableSourceCoverageImpl(JSON.parse(serialized)); // Keep objects referenced by cached modules alive across non-isolated runs. - for (const file of Object.values(globalThis[COVERAGE_KEY] ?? {})) { - for (const metric of ["s", "f"]) { + for (const file of Object.values(coverageGlobal[COVERAGE_KEYImpl] ?? {})) { + for (const metric of ["s", "f"] as const) { for (const key of Object.keys(file[metric])) file[metric][key] = 0; } for (const key of Object.keys(file.b)) file.b[key].fill(0); } } -function canonicalFilename(id) { +function canonicalFilename(id: string) { const filename = id.startsWith("file:") ? fileURLToPath(id) : id.split("?")[0]; return path.resolve(filename).replaceAll("\\", "/"); } -function isIncluded(filename, options, matcher) { +function isIncluded( + filename: string, + options: CoverageOptions, + matcher: (filename: string) => boolean, +) { if ( options.allowExternal === false && !options.roots.some((root) => { @@ -61,15 +118,15 @@ function isIncluded(filename, options, matcher) { return matcher(filename) && (!options.changedFiles || options.changedFiles.includes(filename)); } -function guardSerializedCounters(program, types) { +function guardSerializedCounters(program: ProgramPath, types: BabelTypes) { const factory = program.node.body[0]; if (factory?.loc || !types.isFunctionDeclaration(factory)) return; - const name = factory.id.name; + const name = factory.id!.name; if (!name.startsWith("cov_")) return; program.traverse({ UpdateExpression(counter) { if (counter.node.loc || !counter.getFunctionParent()) return; - let root = counter.node.argument; + let root: import("@babel/types").Node = counter.node.argument; while (types.isMemberExpression(root)) root = root.object; if (!types.isCallExpression(root) || !types.isIdentifier(root.callee, { name })) return; // A serialized function can run outside the instrumented module. Keep @@ -90,10 +147,34 @@ function guardSerializedCounters(program, types) { }); } -function instrumentSource(source, filename, options) { - const { transformSync } = require("@babel/core"); - const { programVisitor, readInitialCoverage } = require("istanbul-lib-instrument"); - let coverage; +function instrumentSourceImpl( + source: string, + filename: string, + options: CoverageOptions, +): InstrumentedSource { + const { transformSync }: typeof import("@babel/core") = require("@babel/core"); + // The published 1.x declarations describe Babel 6 string paths; the locked + // 6.x implementation uses Babel 7 NodePaths and accepts an AST. + const { + programVisitor, + readInitialCoverage, + }: { + programVisitor( + types: BabelTypes, + filename: string, + options: { + coverageVariable: string; + coverageGlobalScope: string; + coverageGlobalScopeFunc: boolean; + ignoreClassMethods?: string[]; + }, + ): { + enter(path: ProgramPath): void; + exit(path: ProgramPath): { fileCoverage: FileCoverage } | undefined; + }; + readInitialCoverage(ast: import("@babel/types").File): { coverageData: FileCoverage } | null; + } = require("istanbul-lib-instrument"); + let coverage: FileCoverage | undefined; const result = transformSync(source, { filename, configFile: false, @@ -106,14 +187,14 @@ function instrumentSource(source, filename, options) { sourceType: "module", plugins: [ "typescript", - ...(filename.endsWith("x") ? ["jsx"] : []), + ...(filename.endsWith("x") ? ["jsx" as const] : []), ["importAttributes", { deprecatedAssertSyntax: true }], ], }, plugins: [ - ({ types }) => { + ({ types }: { types: BabelTypes }) => { const visitor = programVisitor(types, filename, { - coverageVariable: COVERAGE_KEY, + coverageVariable: COVERAGE_KEYImpl, coverageGlobalScope: "globalThis", coverageGlobalScopeFunc: false, ignoreClassMethods: options.ignoreClassMethods, @@ -122,7 +203,7 @@ function instrumentSource(source, filename, options) { visitor: { Program: { enter: visitor.enter, - exit(program) { + exit(program: ProgramPath) { coverage = visitor.exit(program)?.fileCoverage; if (coverage) guardSerializedCounters(program, types); }, @@ -132,12 +213,15 @@ function instrumentSource(source, filename, options) { }, ], }); + if (!result?.ast || typeof result.code !== "string") { + throw new Error(`Source coverage transform produced no code or AST: ${filename}`); + } if (!coverage) return { code: source, coverage: readInitialCoverage(result.ast)?.coverageData }; return { code: result.code, map: result.map, coverage }; } -function shouldInstrumentSource(id) { - const state = globalThis[STATE_KEY]; +function shouldInstrumentSourceImpl(id: string) { + const state = coverageGlobal[STATE_KEY]; return Boolean( state && state.isEnabled() && @@ -146,20 +230,20 @@ function shouldInstrumentSource(id) { ); } -function instrumentSourceForCoverage(source, id) { - if (!shouldInstrumentSource(id)) return undefined; - const state = globalThis[STATE_KEY]; +function instrumentSourceForCoverageImpl(source: string, id: string) { + if (!shouldInstrumentSourceImpl(id)) return undefined; + const state = coverageGlobal[STATE_KEY]!; const filename = canonicalFilename(id); const cached = state.cache.get(filename); if (cached?.source === source) return cached.result; - const result = instrumentSource(source, filename, state.options); + const result = instrumentSourceImpl(source, filename, state.options); state.cache.set(filename, { source, result }); return result; } -function sourceCoverageCacheIdentity(id) { - if (!shouldInstrumentSource(id)) return ""; - const state = globalThis[STATE_KEY]; +function sourceCoverageCacheIdentityImpl(id: string) { + if (!shouldInstrumentSourceImpl(id)) return ""; + const state = coverageGlobal[STATE_KEY]!; if (!state.fingerprint) { const fs = require("node:fs"); const lockfile = path.resolve(__dirname, "../../package-lock.json"); @@ -176,17 +260,17 @@ function sourceCoverageCacheIdentity(id) { .update(JSON.stringify(state.options.ignoreClassMethods ?? [])) .digest("hex"); } - return state.fingerprint; + return state.fingerprint!; } module.exports = { - OPTIONS_ENV, - COVERAGE_KEY, - enableSourceCoverage, - disableSourceCoverage, - resetSourceCoverage, - instrumentSource, - shouldInstrumentSource, - instrumentSourceForCoverage, - sourceCoverageCacheIdentity, + OPTIONS_ENV: OPTIONS_ENVImpl, + COVERAGE_KEY: COVERAGE_KEYImpl, + enableSourceCoverage: enableSourceCoverageImpl, + disableSourceCoverage: disableSourceCoverageImpl, + resetSourceCoverage: resetSourceCoverageImpl, + instrumentSource: instrumentSourceImpl, + shouldInstrumentSource: shouldInstrumentSourceImpl, + instrumentSourceForCoverage: instrumentSourceForCoverageImpl, + sourceCoverageCacheIdentity: sourceCoverageCacheIdentityImpl, }; diff --git a/test/helpers/source-require-compiler.ts b/test/helpers/source-require-compiler.ts index c1225b264d8..9bd4af4bb67 100644 --- a/test/helpers/source-require-compiler.ts +++ b/test/helpers/source-require-compiler.ts @@ -7,7 +7,10 @@ import os from "node:os"; import path from "node:path"; import { performance } from "node:perf_hooks"; import ts from "typescript"; -import { instrumentSourceForCoverage, sourceCoverageCacheIdentity } from "./source-coverage.cts"; +const { + instrumentSourceForCoverage, + sourceCoverageCacheIdentity, +}: typeof import("./source-coverage.cts") = require("./source-coverage.cts"); import { loadSourceRequireCompilerOptions, diff --git a/test/repository/source-require-loader.test.ts b/test/repository/source-require-loader.test.ts index 35f644be1c5..bee2ae4c1a9 100644 --- a/test/repository/source-require-loader.test.ts +++ b/test/repository/source-require-loader.test.ts @@ -567,6 +567,7 @@ const expected = new Set([ path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "register-source-require.ts"))}), path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "source-require-cache.ts"))}), path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "source-require-compiler.ts"))}), + path.resolve(${JSON.stringify(path.join(import.meta.dirname, "../helpers", "source-coverage.cts"))}), ]); const compiled = []; const originalCompile = Module.prototype._compile; @@ -612,7 +613,7 @@ if (require.cache[typescriptPath] !== undefined) { require.cache[typescriptPath] === undefined || registeredTypeScriptLoader === nativeTypeScriptLoader || !rejectedUnexpected || - compiled.length !== 3 || + compiled.length !== expected.size || compiled.some((entry) => !entry.sourceMapped) ) { console.error(JSON.stringify({ fixture, compiled, rejectedUnexpected, typescriptLoaded: require.cache[typescriptPath] !== undefined })); From 39164ed5501f12260326d2d63a8937aecb428408 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 7 Oct 2026 00:20:16 -0700 Subject: [PATCH 3/5] fix(test): bound coverage fixtures and narrow gateway imports Signed-off-by: Prekshi Vyas --- test/e2e/fixtures/clients/host.ts | 2 +- test/repository/source-coverage.test.ts | 56 ++++++++++++------------- 2 files changed, 27 insertions(+), 31 deletions(-) diff --git a/test/e2e/fixtures/clients/host.ts b/test/e2e/fixtures/clients/host.ts index 02dbe365535..82a9460a973 100644 --- a/test/e2e/fixtures/clients/host.ts +++ b/test/e2e/fixtures/clients/host.ts @@ -50,7 +50,7 @@ async function forwardListenerAuthority(env: NodeJS.ProcessEnv): Promise<{ }> { const [ports, gatewayEnv, gatewayIdentity, gatewayManagement] = await Promise.all([ import("../../../../src/lib/core/ports.ts"), - import("../../../../src/lib/onboard/docker-driver-gateway-env.ts"), + import("../../../../src/lib/core/gateway-address.ts"), import("../../../../src/lib/onboard/gateway-binding/identity.ts"), import("../../../../src/lib/onboard/gateway-management.ts"), ]); diff --git a/test/repository/source-coverage.test.ts b/test/repository/source-coverage.test.ts index 7d219bbe620..2830b2f5428 100644 --- a/test/repository/source-coverage.test.ts +++ b/test/repository/source-coverage.test.ts @@ -14,6 +14,7 @@ import { testTimeoutOptions } from "../helpers/timeouts"; const repositoryRoot = path.resolve(import.meta.dirname, "../.."); const roots: string[] = []; +const fixtureDeadlines = new Map(); type Coverage = { fnMap: Record; f: Record; @@ -23,11 +24,13 @@ type Coverage = { afterEach(() => { for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); + fixtureDeadlines.clear(); }); function createFixture(mode: string, floor?: number, project = false) { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-source-coverage-")); roots.push(root); + fixtureDeadlines.set(root, performance.now() + testTimeoutOptions(35_000).timeout - 5_000); fs.mkdirSync(path.join(root, "src")); fs.symlinkSync(path.join(repositoryRoot, "node_modules"), path.join(root, "node_modules"), "dir"); fs.writeFileSync(path.join(root, "package.json"), '{"type":"module"}'); @@ -100,20 +103,28 @@ export default defineConfig({ return root; } -function runFixture(root: string, args = ["run", "--coverage"]) { - return spawnSync( - process.execPath, - [path.join(repositoryRoot, "node_modules/vitest/vitest.mjs"), ...args, "--maxWorkers=1"], - { - cwd: root, - env: { - ...process.env, - NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), - }, - encoding: "utf8", - timeout: 30_000, +function spawnFixture(root: string, args: string[], timeout = 30_000) { + const remaining = Math.floor((fixtureDeadlines.get(root) ?? 0) - performance.now()); + expect(remaining, "Coverage fixture exhausted its shared test budget").toBeGreaterThan(0); + const result = spawnSync(process.execPath, args, { + cwd: root, + env: { + ...process.env, + NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), }, - ); + encoding: "utf8", + timeout: Math.min(timeout, remaining), + }); + expect(result.error, `Coverage fixture subprocess failed: ${args[0]}`).toBeUndefined(); + return result; +} + +function runFixture(root: string, args = ["run", "--coverage"]) { + return spawnFixture(root, [ + path.join(repositoryRoot, "node_modules/vitest/vitest.mjs"), + ...args, + "--maxWorkers=1", + ]); } function functions(coverage: Coverage) { @@ -307,14 +318,7 @@ const probe = require(${JSON.stringify(filename)}); const value = probe.nativeBranch(true); console.log(JSON.stringify({value,coverage:globalThis[coverage.COVERAGE_KEY][${JSON.stringify(filename)}],instrumenters:Object.keys(require.cache).filter(filename => /(?:@babel.core|istanbul-lib-instrument)/.test(filename))})); `; - const result = spawnSync(process.execPath, ["-e", script], { - env: { - ...process.env, - NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), - }, - encoding: "utf8", - timeout: 10_000, - }); + const result = spawnFixture(root, ["-e", script], 10_000); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const output = JSON.parse(result.stdout); expect(output.value).toBe("native-true"); @@ -355,15 +359,7 @@ try { } finally { await ctx.close(); } `, ); - const result = spawnSync(process.execPath, [path.join(root, "toggle.mjs")], { - cwd: root, - env: { - ...process.env, - NODE_OPTIONS: nodeOptionsWithoutSourceLoader(process.env.NODE_OPTIONS), - }, - encoding: "utf8", - timeout: 30_000, - }); + const result = spawnFixture(root, [path.join(root, "toggle.mjs")]); expect(result.status, `${result.stdout}\n${result.stderr}`).toBe(0); const first = JSON.parse(fs.readFileSync(path.join(root, "first.json"), "utf8")); const second = JSON.parse(fs.readFileSync(path.join(root, "second.json"), "utf8")); From bdac003d725b5a21f3c8ce61ba39ebbf6d3c3ff7 Mon Sep 17 00:00:00 2001 From: Rebecca Sliter Date: Wed, 7 Oct 2026 11:30:26 -0700 Subject: [PATCH 4/5] fix(e2e): approve admin scope before portable readiness (#12760) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Outcome Managed-image admin approval now reaches its prepared shell when portable OpenClaw has the expected pending scope upgrade. The fixture uses the trusted OpenShell shell transport directly, so NemoClaw readiness cannot reject the intentionally unsettled request before approval. ## Reason PR #12538 added exact-request filtering but still opened the approval shell through `nemoclaw connect`. Portable readiness correctly rejects an unsettled admin request, so the rootless main workflow failed before the prepared approval ran. ### Related issues Refs #12538. ## Changes - Let the shared approval-script builder accept a shell-quoted connect command and a prepared-shell prelude. The managed-image fixture needs both to approve the pending request before product readiness. The default `nemoclaw connect` path remains unchanged for other consumers. - Run exact-request approval through OpenShell's interactive sandbox shell. This is the existing credential-prepared boundary, while `nemoclaw connect` cannot be its entry point during an intentionally pending approval. - Keep the exact-request filter inside the one-line prepared payload. Integration and E2E-support tests protect exact selection, stale-request exclusion, cleanup, redaction, and the single-line terminal boundary. The live contract remains in `test/e2e/live/managed-image-activation-e2e.test.ts`. Its real boundary is the OpenShell interactive shell with portable pairing pending. It proves that the captured admin request is approved and the downstream OpenClaw operation succeeds. Deterministic payload and selector behavior stays in `test/security/admin-approval-helper.test.ts` and `test/e2e/support/managed-image-activation-diagnostics.test.ts`. ## Verification - `npx vitest run --project integration test/security/admin-approval-helper.test.ts` — 31 tests passed. - `npx vitest run --project e2e-support test/e2e/support/managed-image-activation-diagnostics.test.ts` — 28 tests passed. - `npm run typecheck:cli` — passed. - `npm run validate:pr` — passed. - Normal commit and pre-push hooks — passed, including publication validation and CLI TypeScript checks. - Diff inspection confirmed that no secrets, API keys, or credentials are present. --- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --------- Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com> --- test/e2e/README.md | 11 +-- test/e2e/fixtures/admin-approval-connect.ts | 14 +++- test/e2e/live/openclaw-admin-scope.ts | 68 ++++++++++--------- ...naged-image-activation-diagnostics.test.ts | 27 ++++++-- test/security/admin-approval-helper.test.ts | 57 +++++++++++++--- .../support/admin-approval-connect-fixture.ts | 25 +++++++ 6 files changed, 148 insertions(+), 54 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index e1496ec71ae..9130fafdf37 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -186,11 +186,12 @@ These are two required acceptance executions, not retries; either failure remain OpenClaw feature tests use the shared explicit admin-approval fixture before native operations that require elevated scopes. It approves only the request ID emitted by the current sandbox's non-admin CLI, after the existing selector verifies that device and its requested scopes. -The fixture transfers its approval script through non-terminal `exec --stdin`, then runs the -verified script bytes in a subshell of the prepared `connect` shell. The subshell inherits its -approval wrapper while isolating the script's exit and cleanup trap. This preserves the credential boundary -without feeding a bulk script through terminal line editing. Cleanup removes the temporary -script after success or failure; a cleanup failure also fails the fixture. The fixture checks +The fixture transfers its approval script through non-terminal `exec --stdin`. The exact-request +path then starts an interactive shell through `openshell sandbox exec` for the named gateway and +runs the verified script bytes in that prepared shell's non-interactive interpreter. The +interpreter inherits the approval wrapper while isolating the script's exit and cleanup trap. This +preserves the credential boundary without feeding a bulk script through terminal line editing. +Cleanup removes the temporary script after success or failure; a cleanup failure also fails the fixture. The fixture checks the transferred bytes against the host's digest before evaluation and rejects a replaced script. Managed-image activation retains its cron-consumer proof. Feature setup can stop after the exact approval and verify the grant through its own native operation, avoiding an unrelated cron job or diff --git a/test/e2e/fixtures/admin-approval-connect.ts b/test/e2e/fixtures/admin-approval-connect.ts index 86d84009139..c93aea42a4d 100644 --- a/test/e2e/fixtures/admin-approval-connect.ts +++ b/test/e2e/fixtures/admin-approval-connect.ts @@ -18,6 +18,10 @@ export function adminApprovalConnectScript( cronName: string, expectedRequestId?: string, verifyCronConsumer = true, + options: { + readonly connectCommand?: readonly string[]; + readonly preparedShellPrelude?: string; + } = {}, ): string { const cli = shellQuote(cliPath); const sandbox = shellQuote(sandboxName); @@ -38,7 +42,13 @@ export function adminApprovalConnectScript( // A fresh non-interactive interpreter owns the body's exit and EXIT trap. // Export the prepared approval wrapper; never fall back to the bare CLI. // Keep startup hooks out of this child and pass only the verified bytes. - const connectSuffix = ` ${shellQuote(digest)}) && ( export -f openclaw && BASH_ENV=/dev/null /bin/bash --noprofile --norc <<< "$approval_body" ); approval_status=$?; printf 'ADMIN_CONNECT_BODY_STATUS=%s\\n' "$approval_status"; exit "$approval_status"`; + const preparedShell = options.preparedShellPrelude + ? `( ${options.preparedShellPrelude} BASH_ENV=/dev/null /bin/bash --noprofile --norc <<< "$approval_body" )` + : `( export -f openclaw && BASH_ENV=/dev/null /bin/bash --noprofile --norc <<< "$approval_body" )`; + const connectSuffix = ` ${shellQuote(digest)}) && ${preparedShell}; approval_status=$?; printf 'ADMIN_CONNECT_BODY_STATUS=%s\\n' "$approval_status"; exit "$approval_status"`; + const connectCommand = options.connectCommand + ? options.connectCommand.map(shellQuote).join(" ") + : `${cli} ${sandbox} connect`; return [ "set -euo pipefail", // Connect allocates an interactive terminal. Its line editor can corrupt a @@ -73,7 +83,7 @@ export function adminApprovalConnectScript( // Only the validated generated path is expanded on the host. The script // body and status variables belong to the prepared shell, not this shell. "approval_connect_status=0", - `printf '%s%s%s\n' ${shellQuote(connectPrefix)} "$approval_script" ${shellQuote(connectSuffix)} | ${cli} ${sandbox} connect || approval_connect_status=$?`, + `printf '%s%s%s\n' ${shellQuote(connectPrefix)} "$approval_script" ${shellQuote(connectSuffix)} | ${connectCommand} || approval_connect_status=$?`, 'printf "ADMIN_CONNECT_STATUS=%s\\n" "$approval_connect_status"', 'exit "$approval_connect_status"', ].join("\n"); diff --git a/test/e2e/live/openclaw-admin-scope.ts b/test/e2e/live/openclaw-admin-scope.ts index 05d1ea7fb09..8ddcd2d6046 100644 --- a/test/e2e/live/openclaw-admin-scope.ts +++ b/test/e2e/live/openclaw-admin-scope.ts @@ -4,6 +4,7 @@ import { adminApprovalConnectScript } from "../fixtures/admin-approval-connect.ts"; import { type HostCliClient, type SandboxClient, resultText } from "../fixtures/clients/index.ts"; import { expect } from "../fixtures/e2e-test.ts"; +import { buildCliOpenShellSandboxExecArgs } from "../../../src/lib/adapters/openshell/sandbox-command-cli.ts"; import { shellQuote } from "../../../src/lib/core/shell-quote.ts"; import { pendingAdminRequestId, @@ -13,13 +14,8 @@ import { const AGENT_TIMEOUT_MS = 3 * 60_000; const OPENCLAW_ADMIN_APPROVAL_CAPTURE_LIMIT_BYTES = 64 * 1024; const OPENCLAW_ADMIN_APPROVAL_MARKER = "ISSUE_5324_ADMIN_APPROVAL_OK"; -const EXACT_PENDING_REQUEST_FILTER_PY = `import json, sys -data=json.load(sys.stdin) -pending=data.get("pending") -pending=[] if pending is None else pending -want=str(sys.argv[1] or "").strip() -data["pending"]=[request for request in pending if str(request.get("requestId") or "").strip() == want] -json.dump(data, sys.stdout, separators=(",", ":"))`; +const EXACT_PENDING_REQUEST_FILTER_PY = + 'import json,sys; data=json.load(sys.stdin); pending=data.get("pending"); pending=[] if pending is None else pending; want=str(sys.argv[1] or "").strip(); data["pending"]=[request for request in pending if str(request.get("requestId") or "").strip() == want]; json.dump(data,sys.stdout,separators=(",",":"))'; export function exactRequestAdminApprovalConnectScript( cliPath: string, @@ -27,36 +23,42 @@ export function exactRequestAdminApprovalConnectScript( cronName: string, expectedRequestId: string, verifyCronConsumer = true, + connect: { readonly gatewayName: string; readonly openshellPath: string }, ): string { - const script = adminApprovalConnectScript( - cliPath, - sandboxName, - cronName, - expectedRequestId, - verifyCronConsumer, - ); - const connectPipe = `| ${shellQuote(cliPath)} ${shellQuote(sandboxName)} connect`; + const openshellConnect = [ + connect.openshellPath, + ...buildCliOpenShellSandboxExecArgs({ + command: ["/bin/bash", "-i"], + sandboxName, + target: { kind: "named", gatewayName: connect.gatewayName }, + tty: true, + }), + ]; const connectPrelude = [ - `export NEMOCLAW_ADMIN_EXPECTED_REQUEST_ID=${shellQuote(expectedRequestId)}`, - 'nemoclaw_original_openclaw_definition="$(declare -f openclaw)"', - "case $? in 0) ;; *) exit 33 ;; esac", - 'eval "nemoclaw_original_openclaw${nemoclaw_original_openclaw_definition#openclaw}"', + `export NEMOCLAW_ADMIN_EXPECTED_REQUEST_ID=${shellQuote(expectedRequestId)};`, + 'nemoclaw_original_openclaw_definition="$(declare -f openclaw)";', + "case $? in 0) ;; *) exit 33 ;; esac;", + 'eval "nemoclaw_original_openclaw${nemoclaw_original_openclaw_definition#openclaw}";', "openclaw() {", ' case "$#:${1-}:${2-}:${3-}" in', " 3:devices:list:--json)", - " local -a command_status", - ` nemoclaw_original_openclaw "$@" | python3 -c ${shellQuote(EXACT_PENDING_REQUEST_FILTER_PY)} "$NEMOCLAW_ADMIN_EXPECTED_REQUEST_ID"`, - ' command_status=("${PIPESTATUS[@]}")', - ' case "${command_status[0]}" in 0) return "${command_status[1]}" ;; *) return "${command_status[0]}" ;; esac', + " local -a command_status;", + ` nemoclaw_original_openclaw "$@" | python3 -c ${shellQuote(EXACT_PENDING_REQUEST_FILTER_PY)} "$NEMOCLAW_ADMIN_EXPECTED_REQUEST_ID";`, + ' command_status=("${PIPESTATUS[@]}");', + ' case "${command_status[0]}" in 0) return "${command_status[1]}" ;; *) return "${command_status[0]}" ;; esac;', " ;;", - " esac", - ' nemoclaw_original_openclaw "$@"', - "}", - "export -f nemoclaw_original_openclaw openclaw", - ].join("\n"); - return script.replace( - connectPipe, - `| { printf '%s\\n' ${shellQuote(connectPrelude)}; cat; } ${connectPipe}`, + " esac;", + ' nemoclaw_original_openclaw "$@";', + "};", + "export -f nemoclaw_original_openclaw openclaw;", + ].join(" "); + return adminApprovalConnectScript( + cliPath, + sandboxName, + cronName, + expectedRequestId, + verifyCronConsumer, + { connectCommand: openshellConnect, preparedShellPrelude: connectPrelude }, ); } @@ -106,6 +108,10 @@ export async function approveOpenClawAdminScope( cronName, requestId, verifyCronConsumer, + { + gatewayName: env.OPENSHELL_GATEWAY?.trim() || "nemoclaw", + openshellPath: host.openshellCommandPath, + }, ), ], { diff --git a/test/e2e/support/managed-image-activation-diagnostics.test.ts b/test/e2e/support/managed-image-activation-diagnostics.test.ts index d61e50035dc..7f67a0baff4 100644 --- a/test/e2e/support/managed-image-activation-diagnostics.test.ts +++ b/test/e2e/support/managed-image-activation-diagnostics.test.ts @@ -14,7 +14,10 @@ import { } from "../live/openclaw-admin-scope.ts"; import { adminApprovalConnectScript } from "../fixtures/admin-approval-connect.ts"; import { createHostProcessWorkspace } from "../../helpers/host-process-harness.ts"; -import { ADMIN_APPROVAL_TEST_CLI_SH } from "../../support/admin-approval-connect-fixture.ts"; +import { + ADMIN_APPROVAL_TEST_CLI_SH, + ADMIN_APPROVAL_TEST_OPENSHELL_SH, +} from "../../support/admin-approval-connect-fixture.ts"; import { ArtifactSink } from "../fixtures/artifacts.ts"; import { captureManagedImageOnboardPairingDiagnostics, @@ -291,7 +294,11 @@ describe("managed image activation failure diagnostics", () => { try { await approveOpenClawAdminScope( - { command: hostCommand, commandPath: "/fixture/nemoclaw" } as never, + { + command: hostCommand, + commandPath: "/fixture/nemoclaw", + openshellCommandPath: "/fixture/openshell", + } as never, { exec: sandboxExec } as never, "fixture-sandbox", {}, @@ -354,7 +361,11 @@ describe("managed image activation failure diagnostics", () => { stdout: "ISSUE_5324_ADMIN_APPROVAL_OK\n", timedOut: false, })); - const host = { command: hostCommand, commandPath: "/fixture/nemoclaw" } as never; + const host = { + command: hostCommand, + commandPath: "/fixture/nemoclaw", + openshellCommandPath: "/fixture/openshell", + } as never; const sandbox = { exec: sandboxExec } as never; await approveOpenClawAdminScope(host, sandbox, "fixture-sandbox", {}); @@ -374,6 +385,7 @@ describe("managed image activation failure diagnostics", () => { const staleRequestId = "a96ada31-9cf9-4d99-97cc-978dcbb9fc39"; const commandLog = fixture.path("commands.log"); const cliPath = fixture.writeExecutable("nemoclaw", ADMIN_APPROVAL_TEST_CLI_SH); + const openshellPath = fixture.writeExecutable("openshell", ADMIN_APPROVAL_TEST_OPENSHELL_SH); fs.writeFileSync(join(fixture.homeDir, ".bash_logout"), "echo HOST_LOGOUT_RAN; false\n"); fixture.writeExecutable( "openclaw", @@ -406,7 +418,11 @@ esac return { ...result, exitCode: result.status, timedOut: false }; }); await approveOpenClawAdminScope( - { command: hostCommand, commandPath: cliPath } as never, + { + command: hostCommand, + commandPath: cliPath, + openshellCommandPath: openshellPath, + } as never, { exec: async () => ({ exitCode: 1, @@ -453,13 +469,14 @@ printf '%s\n' "$*" >"$MANAGED_ADMIN_APPROVE_LOG" }; devicesState.pending.push({ ...devicesState.pending[0] }); fs.writeFileSync(devicesPath, JSON.stringify(devicesState)); + const openshellPath = fixture.writeExecutable("openshell", ADMIN_APPROVAL_TEST_OPENSHELL_SH); const result = fixture.run( "/bin/bash", [ "-lc", `PATH=${JSON.stringify(fixture.binDir)}:$PATH export PATH -${exactRequestAdminApprovalConnectScript(cliPath, "fixture-sandbox", "managed-cron", requestId)}`, +${exactRequestAdminApprovalConnectScript(cliPath, "fixture-sandbox", "managed-cron", requestId, true, { gatewayName: "nemoclaw", openshellPath })}`, ], { env: fixture.environment({ diff --git a/test/security/admin-approval-helper.test.ts b/test/security/admin-approval-helper.test.ts index 254a42b817c..72dfc9db749 100644 --- a/test/security/admin-approval-helper.test.ts +++ b/test/security/admin-approval-helper.test.ts @@ -13,6 +13,7 @@ import { ADMIN_REQUEST_SELECTOR_PY } from "../e2e/fixtures/admin-request-selecto import { exactRequestAdminApprovalConnectScript } from "../e2e/live/openclaw-admin-scope.ts"; import { ADMIN_APPROVAL_TEST_CLI_SH, + ADMIN_APPROVAL_TEST_OPENSHELL_SH, ADMIN_APPROVAL_TEST_PTY_PY, removeStagedAdminScript, } from "../support/admin-approval-connect-fixture.ts"; @@ -23,6 +24,18 @@ import { const EXPECTED_REQUEST_ID = "12345678-1234-4123-8123-123456789abc"; const VERSION_ONE_REQUEST_ID = "12345678-1234-1123-8123-123456789abc"; +const EXPECTED_OPENSHELL_ARGS = [ + "sandbox", + "exec", + "--name", + "e2e-issue-4462", + "-g", + "nemoclaw", + "--tty", + "--", + "/bin/bash", + "-i", +]; const EXPECTED_PUBLIC_KEY_BYTES = Buffer.from(Array.from({ length: 32 }, (_value, index) => index)); const EXPECTED_PUBLIC_KEY = EXPECTED_PUBLIC_KEY_BYTES.toString("base64url"); const EXPECTED_DEVICE_ID = createHash("sha256").update(EXPECTED_PUBLIC_KEY_BYTES).digest("hex"); @@ -148,14 +161,17 @@ function runAdminApprovalScript( ): { commands: string[]; capturedApprovalBytes: number; + openshellArgs: string[]; stagedScriptRetained: boolean; result: SpawnSyncReturns; } { const root = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-admin-script-")); const cliPath = path.join(root, "nemoclaw"); + const openshellPath = path.join(root, "openshell"); const openclawPath = path.join(root, "openclaw"); const devicesPath = path.join(root, "devices.json"); const commandLogPath = path.join(root, "openclaw.log"); + const openshellArgvLogPath = path.join(root, "openshell-argv.log"); const mktempCounterPath = path.join(root, "mktemp-counter"); const stagedScriptReceipt = path.join(root, "staged-script"); const stateRoot = writeLocalIdentity(root); @@ -180,6 +196,7 @@ ${options.omitPreparedWrapper ? "unset -f openclaw" : ""} `, ); fs.writeFileSync(cliPath, ADMIN_APPROVAL_TEST_CLI_SH, { mode: 0o755 }); + fs.writeFileSync(openshellPath, ADMIN_APPROVAL_TEST_OPENSHELL_SH, { mode: 0o755 }); fs.writeFileSync( path.join(root, "mktemp"), `#!/bin/sh @@ -244,6 +261,7 @@ esac FAKE_DEVICES_STATE: devicesPath, FAKE_OPENCLAW_FAIL: failureCommand ?? "", FAKE_OPENCLAW_LOG: commandLogPath, + FAKE_OPENSHELL_ARGV_LOG: openshellArgvLogPath, FAKE_FAILURE_OUTPUT_PADDING_BYTES: String(failureOutputPaddingBytes), FAKE_MKTEMP_COUNTER: mktempCounterPath, FAKE_MKTEMP_ROOT: root, @@ -271,6 +289,7 @@ esac "admin-cron", options.expectedRequestId ?? requestId, options.verifyCronConsumer, + { gatewayName: "nemoclaw", openshellPath }, ) : adminApprovalConnectScript( cliPath, @@ -287,10 +306,13 @@ esac const capturedApprovalBytes = fs.existsSync(approvalCapturePath) ? fs.statSync(approvalCapturePath).size : 0; + const openshellArgs = fs.existsSync(openshellArgvLogPath) + ? fs.readFileSync(openshellArgvLogPath, "utf8").trim().split("\n") + : []; const stagedScriptRetained = fs.existsSync(stagedScriptReceipt) && fs.existsSync(fs.readFileSync(stagedScriptReceipt, "utf8")); - return { capturedApprovalBytes, commands, result, stagedScriptRetained }; + return { capturedApprovalBytes, commands, openshellArgs, result, stagedScriptRetained }; } finally { removeStagedAdminScript(stagedScriptReceipt); fs.rmSync(root, { recursive: true, force: true }); @@ -387,14 +409,20 @@ describe("prepared connect-shell administrative approval", () => { it("approves the captured request when another request remains pending", () => { const staleRequestId = "87654321-4321-4321-8321-cba987654321"; - const { commands, result } = runAdminApprovalScript(undefined, 0, EXPECTED_REQUEST_ID, { - expectedRequestId: EXPECTED_REQUEST_ID, - pendingRequestIds: [staleRequestId, EXPECTED_REQUEST_ID], - selectExpectedRequest: true, - verifyCronConsumer: false, - }); + const { commands, openshellArgs, result } = runAdminApprovalScript( + undefined, + 0, + EXPECTED_REQUEST_ID, + { + expectedRequestId: EXPECTED_REQUEST_ID, + pendingRequestIds: [staleRequestId, EXPECTED_REQUEST_ID], + selectExpectedRequest: true, + verifyCronConsumer: false, + }, + ); expect(result.status, result.stderr).toBe(0); + expect(openshellArgs).toEqual(EXPECTED_OPENSHELL_ARGS); expect(commands).toEqual(["devices list --json", `devices approve ${EXPECTED_REQUEST_ID}`]); expect(result.stdout).toContain("ISSUE_5324_ADMIN_APPROVAL_OK"); expect(`${result.stdout}\n${result.stderr}`).not.toContain(staleRequestId); @@ -494,12 +522,19 @@ describe("prepared connect-shell administrative approval", () => { ] as const)( "binds feature approval to request %s without running cron", (expectedRequestId, status) => { - const { commands, result } = runAdminApprovalScript("cron:add", 0, EXPECTED_REQUEST_ID, { - expectedRequestId, - verifyCronConsumer: false, - }); + const { commands, openshellArgs, result } = runAdminApprovalScript( + "cron:add", + 0, + EXPECTED_REQUEST_ID, + { + expectedRequestId, + verifyCronConsumer: false, + selectExpectedRequest: true, + }, + ); expect(result.status, result.stderr).toBe(status); + expect(openshellArgs).toEqual(EXPECTED_OPENSHELL_ARGS); expect(commands).toEqual([ "devices list --json", ...(status === 0 ? [`devices approve ${EXPECTED_REQUEST_ID}`] : []), diff --git a/test/support/admin-approval-connect-fixture.ts b/test/support/admin-approval-connect-fixture.ts index 437e801ba6b..38843be1c05 100644 --- a/test/support/admin-approval-connect-fixture.ts +++ b/test/support/admin-approval-connect-fixture.ts @@ -31,6 +31,31 @@ case "$2" in esac `; +export const ADMIN_APPROVAL_TEST_OPENSHELL_SH = `#!/bin/sh +set -eu +if [ -n "\${FAKE_OPENSHELL_ARGV_LOG:-}" ]; then + : >"$FAKE_OPENSHELL_ARGV_LOG" + for arg in "$@"; do printf '%s\\n' "$arg" >>"$FAKE_OPENSHELL_ARGV_LOG"; done +fi +[ "$1" = sandbox ] +[ "$2" = exec ] +shift 2 +while [ "$1" != -- ]; do shift; done +shift +[ "$1" = /bin/bash ] +[ "$2" = -i ] +approval_program="$(cat)" +case "$approval_program" in + *' +'*) echo "ADMIN_CONNECT_INPUT_NOT_SINGLE_LINE" >&2; exit 96 ;; +esac +if [ -n "\${ADMIN_CONNECT_TERMINAL_PROBE:-}" ]; then + printf '%s\n' "$approval_program" | python3 "$ADMIN_CONNECT_TERMINAL_PROBE" + exit $? +fi +printf '%s\n' "$approval_program" | /bin/bash -c 'openclaw() { command openclaw "$@"; }; if [ -n "\${ADMIN_CONNECT_RC:-}" ]; then . "$ADMIN_CONNECT_RC"; fi; . /dev/stdin' +`; + export const ADMIN_APPROVAL_TEST_PTY_PY = ` """Run the fixture's connect input through a real interactive shell terminal.""" From 3b52b9ae4bf127bc327389fd4b944eedb5a84052 Mon Sep 17 00:00:00 2001 From: Prekshi Vyas Date: Wed, 7 Oct 2026 14:53:57 -0700 Subject: [PATCH 5/5] test(ci): cover bounded pinned search-tool retries Signed-off-by: Prekshi Vyas --- .../pr-workflow-contract.test.ts | 70 +++++++++++++++++-- 1 file changed, 63 insertions(+), 7 deletions(-) diff --git a/test/automation/pull-requests/pr-workflow-contract.test.ts b/test/automation/pull-requests/pr-workflow-contract.test.ts index 888cbeb16b3..237c56caa32 100644 --- a/test/automation/pull-requests/pr-workflow-contract.test.ts +++ b/test/automation/pull-requests/pr-workflow-contract.test.ts @@ -512,13 +512,69 @@ printf '%s %s\\n' '6bf226944684f56c84dd014e8b979d27425c0148f61b3bd99bcc6f39e9dc ); }, ); - it.each([ - ["CLI shards", requiredStep(sharedActions.cliCoverageShard, "Install pinned Pi search tools")], - [ - "Advisor runtime", - requiredWorkflowStep(advisorWorkflow.jobs["build-advisor-runtime"], "Install locked runtime"), - ], - ])("refreshes only Ubuntu package metadata for %s", (_name, installStep) => { + it("bounds and retries only Ubuntu package metadata for CLI shards", () => { + const installStep = requiredStep( + sharedActions.cliCoverageShard, + "Install pinned Pi search tools", + ); + const temp = mkdtempSync(join(tmpdir(), "nemoclaw-ubuntu-apt-sources-")); + const fakeBin = join(temp, "bin"); + const aptArgs = join(temp, "apt-args"); + mkdirSync(fakeBin); + writeFileSync( + join(fakeBin, "sudo"), + '#!/usr/bin/env bash\nprintf "%s\\n" "$@" "__END_CALL__" >> "$APT_ARGS"\nexit 86\n', + { mode: 0o755 }, + ); + + try { + const result = runWorkflowShellStep(installStep, { + APT_ARGS: aptArgs, + PATH: `${fakeBin}:${process.env.PATH ?? ""}`, + }); + const expectedCall = [ + "timeout", + "--signal=TERM", + "--kill-after=10s", + "120s", + "apt-get", + "-o", + "Acquire::Retries=2", + "-o", + "Acquire::http::Timeout=20", + "-o", + "Acquire::https::Timeout=20", + "update", + "-o", + "Dir::Etc::sourcelist=sources.list.d/ubuntu.sources", + "-o", + "Dir::Etc::sourceparts=-", + "__END_CALL__", + ]; + + expect(result.status, result.stdout + result.stderr).toBe(1); + expect(result.stdout).toContain( + "Retrying pinned Pi search-tool package index update (attempt 2/3)", + ); + expect(result.stdout).toContain( + "Retrying pinned Pi search-tool package index update (attempt 3/3)", + ); + expect(result.stdout).toContain( + "::error::Could not update pinned Pi search-tool package indexes after three bounded attempts", + ); + expect(readFileSync(aptArgs, "utf8").trim().split("\n")).toEqual( + Array.from({ length: 3 }, () => expectedCall).flat(), + ); + } finally { + rmSync(temp, { force: true, recursive: true }); + } + }); + + it("refreshes only Ubuntu package metadata for Advisor runtime", () => { + const installStep = requiredWorkflowStep( + advisorWorkflow.jobs["build-advisor-runtime"], + "Install locked runtime", + ); const temp = mkdtempSync(join(tmpdir(), "nemoclaw-ubuntu-apt-sources-")); const fakeBin = join(temp, "bin"); const aptArgs = join(temp, "apt-args");