diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f8a554..de3e929 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,3 +19,10 @@ jobs: - run: npm ci - run: npm run check - run: npm test + - run: npm run test:consumer + timeout-minutes: 10 + - uses: actions/upload-artifact@v4 + if: always() + with: + name: isolated-consumer-evidence + path: consumer-results/ diff --git a/.gitignore b/.gitignore index 8300c67..bded4eb 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ coverage/ .gitnexus/ graphify-out/ .claude/settings.local.json + +consumer-results/ diff --git a/package.json b/package.json index f748939..f206792 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,8 @@ "check": "biome check . && tsc --noEmit", "test": "node --import tsx --test 'test/**/*.test.ts'", "prepare": "npm run build && husky", - "prepublishOnly": "npm run check && npm test" + "prepublishOnly": "npm run check && npm test", + "test:consumer": "node --import tsx --test --test-concurrency=1 test/consumer/*.consumer.ts" }, "dependencies": { "@wdio/cli": "^9.29.1", diff --git a/test/consumer/README.md b/test/consumer/README.md new file mode 100644 index 0000000..a453eb8 --- /dev/null +++ b/test/consumer/README.md @@ -0,0 +1,16 @@ +Run `npm run test:consumer` after `npm ci`. The harness builds and packs the +current checkout, records its SHA and tarball SHA256 in consumer-results, and +installs into a fresh OS temporary directory with its own manifest and +prefix and NODE_PATH removed. No producer node_modules is linked. + +This proves CLI help, iOS scaffolding, APK onboarding and public declaration +imports work after installation. Two mutated packages prove missing tsx and +missing onboarding entry failures are detected by their specific diagnostics. +Registry/network failures are UNAVAILABLE and fail the run. Logs are retained +as CI artifacts; temporary consumers are removed even on failure. + +This does not exercise Appium, device connections or application behaviour. +Dependency lifecycle scripts are disabled during installation: this checks the +packed JavaScript and types, not dependency setup hooks or driver downloads. +The nested npm install strategy prevents hoisted transitive dependencies from +masking a missing direct runtime dependency (notably tsx). diff --git a/test/consumer/fault-proofs.consumer.ts b/test/consumer/fault-proofs.consumer.ts new file mode 100644 index 0000000..f5864ef --- /dev/null +++ b/test/consumer/fault-proofs.consumer.ts @@ -0,0 +1,32 @@ +import { test } from "node:test"; +import { faultProof } from "./harness.js"; + +test("missing runtime tsx is detected", () => faultProof("tsx")); +test("missing onboard entry is detected", () => faultProof("onboard")); + +import assert from "node:assert/strict"; +import path from "node:path"; +import { consumer, run } from "./harness.js"; + +test("missing onboard command has a package diagnostic", () => { + const fixture = consumer("onboard"); + try { + assert.throws( + () => run(path.join(fixture.cwd, "node_modules/.bin/nativeproof-onboard"), ["--help"], fixture.cwd), + (error: unknown) => + error instanceof Error && + !error.message.includes("UNAVAILABLE") && + /nativeproof-onboard/.test(error.message), + ); + } finally { + fixture.cleanup(); + } +}); +test("local spawn errors are ordinary failures", () => { + assert.throws( + () => run("/nonexistent-nativeproof-command", [], process.cwd()), + (error: unknown) => error instanceof Error && !error.message.includes("UNAVAILABLE"), + ); +}); + +test("missing public declarations are detected", () => faultProof("types")); diff --git a/test/consumer/harness.ts b/test/consumer/harness.ts new file mode 100644 index 0000000..7a8d654 --- /dev/null +++ b/test/consumer/harness.ts @@ -0,0 +1,118 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { appendFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +const root = process.cwd(); +const logs = path.join(root, "consumer-results"); +mkdirSync(logs, { recursive: true }); +export function run(command: string, args: string[], cwd: string, failure = false) { + const env = { ...process.env }; + delete env.NODE_PATH; + if (cwd !== root) { + env.npm_config_prefix = path.join(cwd, ".prefix"); + } + const result = spawnSync(command, args, { cwd, env, encoding: "utf8", timeout: 240000 }); + const output = `${result.stdout ?? ""}\n${result.stderr ?? ""}`; + appendFileSync(path.join(logs, `${process.pid}.log`), `${command} ${args.join(" ")}\n${output}\n`); + if (/EAI_AGAIN|ENOTFOUND|ECONNRESET|ECONNREFUSED|ETIMEDOUT|E401|E403|E404|E502|E503/.test(output)) { + throw new Error(`UNAVAILABLE: ${output}`); + } + if (result.error) throw new Error(`${command}: ${result.error.message}`); + if (!failure) assert.equal(result.status, 0, output); + return { status: result.status, output, stdout: result.stdout }; +} +export function consumer(fault?: "tsx" | "onboard" | "types") { + const scratch = mkdtempSync(path.join(tmpdir(), "nativeproof-consumer-")); + try { + run("npm", ["run", "build"], root); + const packed = JSON.parse( + run("npm", ["pack", "--json", "--pack-destination", scratch], root).stdout.trim(), + ) as { filename: string }[]; + assert.ok(packed[0]); + let tarball = path.join(scratch, packed[0].filename); + if (fault) { + run("tar", ["-xzf", tarball, "-C", scratch], scratch); + const pkgPath = path.join(scratch, "package", "package.json"); + const pkg = JSON.parse(readFileSync(pkgPath, "utf8")); + if (fault === "tsx") { + delete pkg.dependencies.tsx; + writeFileSync(pkgPath, JSON.stringify(pkg)); + } else + rmSync( + path.join(scratch, "package", "dist", fault === "types" ? "index.d.ts" : "nativeproof-onboard.js"), + ); + tarball = path.join(scratch, "broken.tgz"); + run("tar", ["-czf", tarball, "package"], scratch); + } + const digest = createHash("sha256").update(readFileSync(tarball)).digest("hex"); + writeFileSync(path.join(logs, `${process.pid}-${fault ?? "installed"}.sha256`), `${digest}\n`); + writeFileSync(path.join(logs, `${process.pid}.sha`), run("git", ["rev-parse", "HEAD"], root).output); + const cwd = path.join(scratch, "consumer"); + mkdirSync(cwd); + writeFileSync(path.join(cwd, "package.json"), JSON.stringify({ private: true, type: "module" })); + run( + "npm", + [ + "install", + "--install-strategy=nested", + "--ignore-scripts", + "--no-audit", + "--no-fund", + tarball, + "typescript@5.9.3", + "@types/node@24", + ], + cwd, + ); + return { cwd, cleanup: () => rmSync(scratch, { recursive: true, force: true }) }; + } catch (error) { + rmSync(scratch, { recursive: true, force: true }); + throw error; + } +} +export function faultProof(fault: "tsx" | "onboard" | "types") { + const fixture = consumer(fault); + try { + if (fault === "types") { + writeFileSync( + path.join(fixture.cwd, "import.ts"), + 'import { defineConfig } from "nativeproof"; type Config = Parameters[0];', + ); + const result = run( + path.join(fixture.cwd, "node_modules/.bin/tsc"), + ["--noEmit", "--strict", "--module", "NodeNext", "--target", "ES2022", "import.ts"], + fixture.cwd, + true, + ); + assert.notEqual(result.status, 0); + assert.match(result.output, /TS7016.*nativeproof/); + return; + } + if (fault === "onboard") { + assert.throws( + () => run(path.join(fixture.cwd, "node_modules/.bin/nativeproof-onboard"), ["--help"], fixture.cwd), + /nativeproof-onboard/, + ); + return; + } + const entry = "cli.js"; + const result = run( + process.execPath, + [path.join(fixture.cwd, "node_modules/nativeproof/dist", entry), "--help"], + fixture.cwd, + true, + ); + assert.notEqual(result.status, 0, "broken package must fail"); + assert.match( + result.output, + fault === "tsx" + ? /Cannot find (?:package|module) 'tsx'/ + : /Cannot find module .*nativeproof-onboard\.js/, + ); + } finally { + fixture.cleanup(); + } +} diff --git a/test/consumer/installed.consumer.ts b/test/consumer/installed.consumer.ts new file mode 100644 index 0000000..779317a --- /dev/null +++ b/test/consumer/installed.consumer.ts @@ -0,0 +1,28 @@ +import assert from "node:assert/strict"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import path from "node:path"; +import { test } from "node:test"; +import { consumer, run } from "./harness.js"; + +test("packed CLI and declarations work in an independent install", () => { + const fixture = consumer(); + const { cwd } = fixture; + try { + const bin = (name: string) => path.join(cwd, "node_modules/.bin", name); + assert.match(run(bin("nativeproof"), ["--help"], cwd).output, /nativeproof/); + run(bin("nativeproof-init"), ["--ios"], cwd); + assert.match(readFileSync(path.join(cwd, "nativeproof.config.ts"), "utf8"), /ios/i); + const android = path.join(cwd, "android"); + mkdirSync(android); + writeFileSync(path.join(android, "Example.apk"), ""); + run(bin("nativeproof-onboard"), ["./Example.apk"], android); + assert.match(readFileSync(path.join(android, "nativeproof.config.ts"), "utf8"), /Example\.apk/); + writeFileSync( + path.join(cwd, "import.ts"), + 'import { defineConfig } from "nativeproof"; type Config = Parameters[0]; const config: Config = { projects: [] }; void config;\n', + ); + run(bin("tsc"), ["--noEmit", "--strict", "--module", "NodeNext", "--target", "ES2022", "import.ts"], cwd); + } finally { + fixture.cleanup(); + } +});