From 65d93b3b28f9ec7ecddfde9de15a226379967afd Mon Sep 17 00:00:00 2001 From: Ben Sheridan-Edwards Date: Tue, 15 Sep 2026 15:23:37 +0100 Subject: [PATCH 1/4] feat(evidence): expose failure artifact records (#152) --- README.md | 4 ++ src/config.ts | 54 ++++++++++++++-- src/evidence.ts | 39 +++++++++--- src/index.ts | 2 + test/config.test.ts | 146 +++++++++++++++++++++++++++++++++++--------- test/index.test.ts | 4 ++ 6 files changed, 208 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 2c02094..cbac04d 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,10 @@ export default defineConfig({ }, specFileRetries: 2, // retry a failed spec file reporters: ["spec", ["junit", { outputDir: "reports" }]], + async onFailureEvidence({ project, file, fullName, attempt, pngPath, xmlPath }) { + // Attach or index the exact files NativeProof successfully saved for this attempt. + console.log({ project, file, fullName, attempt, pngPath, xmlPath }); + }, async afterTest(test, _context, result) { // NativeProof still captures its default failure evidence before this hook runs. console.log(`${test.parent} > ${test.title}: ${result.passed ? "passed" : "failed"}`); diff --git a/src/config.ts b/src/config.ts index 16b85fc..3a6b33c 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,12 @@ import { existsSync } from "node:fs"; import path from "node:path"; import type { Frameworks, Reporters, Services } from "@wdio/types"; import type { App } from "./app.js"; -import { captureState, failureEvidenceName, setArtifactDir } from "./evidence.js"; +import { + type CapturedStatePaths, + captureStatePaths, + failureEvidenceName, + setArtifactDir, +} from "./evidence.js"; /** * The Playwright-style config: one `nativeproof.config.ts` declares the app, the device @@ -167,6 +172,19 @@ export type RunnerTest = Frameworks.Test; /** WebdriverIO's complete runner result payload, including retries, duration, status, and error. */ export type RunnerTestResult = Frameworks.TestResult; +/** Exact files produced by NativeProof's built-in capture for one failed test attempt. */ +export interface FailureEvidence { + project: string; + file: string; + fullName: string; + attempt: number; + pngPath: string; + xmlPath: string; +} + +/** Reporter-neutral notification after built-in failure evidence is saved successfully. */ +export type FailureEvidenceCallback = (evidence: FailureEvidence) => unknown | Promise; + /** Consumer hook composed after NativeProof's built-in failure evidence capture. */ export type RunnerAfterTestHook = NonNullable; @@ -196,6 +214,8 @@ export interface RunnerConfig { reporters?: RunnerReporter[]; /** Consumer lifecycle hook; failure capture still runs first. */ afterTest?: RunnerAfterTestHook; + /** Called with the exact built-in PNG/XML files after a failed test capture succeeds. */ + onFailureEvidence?: FailureEvidenceCallback; /** * WebdriverIO pass-throughs for tuning real-device runs. Each is forwarded only when set, so * WebdriverIO's own defaults apply otherwise. Slow software-GPU emulators in particular often @@ -303,16 +323,37 @@ export function projectCapabilities(config: RunnerConfig, project: DeviceProject }; } -type FailureCapture = (prefix: string) => Promise; +type FailureCapture = (prefix: string) => Promise; /** @internal Compose public hooks without allowing them to replace default evidence capture. */ export function composeAfterTest( consumerHook?: RunnerAfterTestHook, - captureFailure: FailureCapture = captureState, + captureFailure: FailureCapture = captureStatePaths, + evidence: { project?: string; onFailureEvidence?: FailureEvidenceCallback } = {}, ): RunnerAfterTestHook { return async (test, context, result) => { if (!result.passed) { - await captureFailure(failureEvidenceName(test)).catch(() => {}); + const attempt = result.retries?.attempts ?? 0; + const identity = { + project: evidence.project ?? "", + file: test.file ?? "", + fullName: test.fullName ?? `${test.parent} ${test.title}`, + attempt, + parent: test.parent, + title: test.title, + }; + await captureFailure(failureEvidenceName(identity)) + .then(async (paths) => { + await evidence.onFailureEvidence?.({ + project: identity.project, + file: identity.file, + fullName: identity.fullName, + attempt, + pngPath: paths.pngPath, + xmlPath: paths.xmlPath, + }); + }) + .catch(() => {}); } await consumerHook?.(test, context, result); }; @@ -353,7 +394,10 @@ export function buildWdioConfig( // redacted page source into the artifact dir, named after the spec. Best-effort — a // capture error never masks the real failure — and consumers get it without writing // their own afterTest hook. - afterTest: composeAfterTest(config.afterTest), + afterTest: composeAfterTest(config.afterTest, captureStatePaths, { + project: project.name, + ...(config.onFailureEvidence ? { onFailureEvidence: config.onFailureEvidence } : {}), + }), }; // Optional WebdriverIO tuning — forwarded only when the consumer set it, so wdio's defaults apply // otherwise (real emulators/simulators often need longer connection/wait timeouts than the defaults). diff --git a/src/evidence.ts b/src/evidence.ts index 3edc480..e369ac7 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -65,16 +65,26 @@ export async function captureScreenshot(filename: string): Promise { return target; } -/** Capture a screenshot + redacted source pair under one prefix; returns the source. */ -export async function captureState(prefix: string): Promise { +export interface CapturedStatePaths { + pngPath: string; + xmlPath: string; +} + +/** @internal Capture a screenshot + redacted source pair and return only paths written successfully. */ +export async function captureStatePaths(prefix: string): Promise { const source = await browser.getPageSource().catch((err: unknown) => { // A failed capture must not look like a clean empty screen in the evidence trail. console.warn(`[nativeproof] getPageSource failed during captureState("${prefix}"): ${err}`); return ""; }); - await captureScreenshot(`${prefix}.png`); - await captureText(`${prefix}.xml`, source); - return source; + const pngPath = await captureScreenshot(`${prefix}.png`); + const xmlPath = await captureText(`${prefix}.xml`, source); + return { source, pngPath, xmlPath }; +} + +/** Capture a screenshot + redacted source pair under one prefix; returns the source. */ +export async function captureState(prefix: string): Promise { + return (await captureStatePaths(prefix)).source; } /** @@ -83,8 +93,23 @@ export async function captureState(prefix: string): Promise { * truncation collisions, and capped at 120 chars. Used by the runner's built-in on-failure * capture so a failing spec leaves a screenshot + source pair with no per-spec wiring. */ -export function failureEvidenceName(test: { parent: string; title: string }): string { +export function failureEvidenceName(test: { + parent: string; + title: string; + project?: string; + file?: string; + fullName?: string; + attempt?: number; +}): string { const raw = `failure-${test.parent}-${test.title}`; - const suffix = `-${createHash("sha1").update(raw).digest("hex").slice(0, 8)}`; + const identity = [ + test.project ?? "", + test.file ?? "", + test.fullName ?? "", + String(test.attempt ?? 0), + test.parent, + test.title, + ].join("\0"); + const suffix = `-${createHash("sha1").update(identity).digest("hex").slice(0, 8)}`; return `${raw.replace(/[^\w.-]+/g, "_").slice(0, 120 - suffix.length)}${suffix}`; } diff --git a/src/index.ts b/src/index.ts index 99a5efa..a970714 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,8 @@ export { defineApp } from "./app.js"; export type { AppiumOptions, DeviceProject, + FailureEvidence, + FailureEvidenceCallback, NativeProofConfig, RunnerAfterTestHook, RunnerConfig, diff --git a/test/config.test.ts b/test/config.test.ts index 69937e0..ae4aed8 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; @@ -300,7 +300,8 @@ test("an explicit empty CLI grep clears config grep without replacing other Moch }); }); -test("composeAfterTest captures failures before the consumer hook and keeps capture best-effort", async () => { +test("composeAfterTest reports exact saved failure evidence before the consumer hook", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-failure-evidence-")); const events: string[] = []; const runnerTest = { type: "test", @@ -309,45 +310,116 @@ test("composeAfterTest captures failures before the consumer hook and keeps capt fullTitle: "suite fails", fullName: "suite fails", pending: false, - file: "tests/failure.spec.ts", + file: "/proj/tests/login.spec.ts", ctx: {}, }; + try { + const pngPath = path.join(dir, "failure.png"); + const xmlPath = path.join(dir, "failure.xml"); + const records: unknown[] = []; + const hook = composeAfterTest( + async () => { + events.push("consumer"); + }, + async () => { + events.push("capture"); + writeFileSync(pngPath, "png"); + writeFileSync(xmlPath, ""); + return { pngPath, xmlPath }; + }, + { + project: "ios", + onFailureEvidence(record) { + events.push("evidence"); + records.push(record); + }, + }, + ); + + await hook( + runnerTest, + {}, + { + passed: false, + duration: 1, + retries: { limit: 1, attempts: 1 }, + exception: "failure", + status: "failed", + }, + ); + assert.deepEqual(events, ["capture", "evidence", "consumer"]); + assert.deepEqual(records, [ + { + project: "ios", + file: "/proj/tests/login.spec.ts", + fullName: "suite fails", + attempt: 1, + pngPath, + xmlPath, + }, + ]); + assert.equal(existsSync(pngPath), true); + assert.equal(existsSync(xmlPath), true); + + events.length = 0; + await hook( + { ...runnerTest, title: "passes", fullTitle: "suite passes", fullName: "suite passes" }, + {}, + { + passed: true, + duration: 1, + retries: { limit: 1, attempts: 0 }, + exception: "", + status: "passed", + }, + ); + assert.deepEqual(events, ["consumer"]); + assert.equal(records.length, 1, "passing tests emit no failure-evidence callback"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("composeAfterTest reports no fictitious paths when capture fails and preserves the failure", async () => { + const events: string[] = []; + const runnerTest = { + type: "test", + title: "fails", + parent: "suite", + fullTitle: "suite fails", + fullName: "suite fails", + pending: false, + file: "/proj/fail.spec.ts", + ctx: {}, + }; + const result = { + passed: false, + duration: 1, + retries: { limit: 0, attempts: 0 }, + exception: "failure", + status: "failed", + }; + let consumerResult: unknown; const hook = composeAfterTest( - async () => { + async (_test, _context, receivedResult) => { events.push("consumer"); + consumerResult = receivedResult; }, async () => { events.push("capture"); throw new Error("device disconnected during evidence capture"); }, - ); - - await hook( - runnerTest, - {}, { - passed: false, - duration: 1, - retries: { limit: 0, attempts: 0 }, - exception: "failure", - status: "failed", + project: "android", + onFailureEvidence() { + events.push("evidence"); + }, }, ); - assert.deepEqual(events, ["capture", "consumer"]); - events.length = 0; - await hook( - { ...runnerTest, title: "passes", fullTitle: "suite passes", fullName: "suite passes" }, - {}, - { - passed: true, - duration: 1, - retries: { limit: 0, attempts: 0 }, - exception: "", - status: "passed", - }, - ); - assert.deepEqual(events, ["consumer"]); + await hook(runnerTest, {}, result); + assert.deepEqual(events, ["capture", "consumer"]); + assert.equal(consumerResult, result); }); test("buildWdioConfig lets nativeproof.config.ts own the artifact directory", async () => { @@ -384,7 +456,7 @@ test("buildWdioConfig adds a built-in evidence-on-failure afterTest hook", async await afterTest({ title: "t", parent: "p" }, {}, { passed: false }); }); -test("failureEvidenceName builds a filesystem-safe, capped and unique prefix", () => { +test("failureEvidenceName is safe, capped and unique across file, project and retry identity", () => { assert.match( failureEvidenceName({ parent: "Room · onboarding", title: "Accept is inert!" }), /^failure-Room_onboarding-Accept_is_inert_-[a-f0-9]{8}$/, @@ -394,6 +466,22 @@ test("failureEvidenceName builds a filesystem-safe, capped and unique prefix", ( assert.ok(first.length <= 120); assert.ok(second.length <= 120); assert.notEqual(first, second); + + const testIdentity = { + parent: "Login", + title: "rejects bad password", + fullName: "Login rejects bad password", + file: "/proj/tests/login.spec.ts", + project: "android", + attempt: 0, + }; + const identities = [ + testIdentity, + { ...testIdentity, file: "/proj/tests/admin-login.spec.ts" }, + { ...testIdentity, project: "ios" }, + { ...testIdentity, attempt: 1 }, + ]; + assert.equal(new Set(identities.map(failureEvidenceName)).size, identities.length); }); test("findConfigFile locates nativeproof.config.* via the injected exists check", () => { diff --git a/test/index.test.ts b/test/index.test.ts index 195ae0f..58646d1 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -15,6 +15,8 @@ import { defineApp, defineConfig, expect, + type FailureEvidence, + type FailureEvidenceCallback, type FailureInfo, type FlowContext, type FrameDirection, @@ -60,6 +62,8 @@ type PublicTypeSmoke = { appium: AppiumOptions; device: DeviceContext; driver: Driver; + failureEvidence: FailureEvidence; + failureEvidenceCallback: FailureEvidenceCallback; failure: FailureInfo; flow: FlowContext; frameDirection: FrameDirection; From c3121c2d23efee8396d4153bcd1d4900a6d328d8 Mon Sep 17 00:00:00 2001 From: Ben Sheridan-Edwards Date: Tue, 15 Sep 2026 15:58:53 +0100 Subject: [PATCH 2/4] fix(evidence): make failure records collision-safe --- src/evidence.ts | 34 ++++++++++++++------ test/config.test.ts | 77 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 12 deletions(-) diff --git a/src/evidence.ts b/src/evidence.ts index e369ac7..f44548b 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -70,15 +70,29 @@ export interface CapturedStatePaths { xmlPath: string; } +interface CaptureStateDependencies { + getPageSource(): Promise; + captureScreenshot(filename: string): Promise; + captureText(filename: string, contents: string): Promise; +} + +const defaultCaptureStateDependencies: CaptureStateDependencies = { + getPageSource: () => browser.getPageSource(), + captureScreenshot, + captureText, +}; + /** @internal Capture a screenshot + redacted source pair and return only paths written successfully. */ -export async function captureStatePaths(prefix: string): Promise { - const source = await browser.getPageSource().catch((err: unknown) => { - // A failed capture must not look like a clean empty screen in the evidence trail. - console.warn(`[nativeproof] getPageSource failed during captureState("${prefix}"): ${err}`); - return ""; - }); - const pngPath = await captureScreenshot(`${prefix}.png`); - const xmlPath = await captureText(`${prefix}.xml`, source); +export async function captureStatePaths( + prefix: string, + dependencies: CaptureStateDependencies = defaultCaptureStateDependencies, +): Promise { + // Reject the whole pair when source capture fails: an empty XML file would make a partial + // capture look complete to structured evidence consumers. composeAfterTest keeps this + // best-effort and preserves the original test failure + consumer hook invocation. + const source = await dependencies.getPageSource(); + const pngPath = await dependencies.captureScreenshot(`${prefix}.png`); + const xmlPath = await dependencies.captureText(`${prefix}.xml`, source); return { source, pngPath, xmlPath }; } @@ -89,7 +103,7 @@ export async function captureState(prefix: string): Promise { /** * A filesystem-safe evidence prefix for a failed behaviour — `failure--` - * with runs of non-word characters collapsed to `_`, a short stable suffix to avoid + * with runs of non-word characters collapsed to `_`, a full stable digest to avoid * truncation collisions, and capped at 120 chars. Used by the runner's built-in on-failure * capture so a failing spec leaves a screenshot + source pair with no per-spec wiring. */ @@ -110,6 +124,6 @@ export function failureEvidenceName(test: { test.parent, test.title, ].join("\0"); - const suffix = `-${createHash("sha1").update(identity).digest("hex").slice(0, 8)}`; + const suffix = `-${createHash("sha256").update(identity).digest("hex")}`; return `${raw.replace(/[^\w.-]+/g, "_").slice(0, 120 - suffix.length)}${suffix}`; } diff --git a/test/config.test.ts b/test/config.test.ts index ae4aed8..473c385 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -13,7 +13,7 @@ import { resolveProject, splitSpecGlobs, } from "../src/config.js"; -import { captureText, failureEvidenceName, setArtifactDir } from "../src/evidence.js"; +import { captureStatePaths, captureText, failureEvidenceName, setArtifactDir } from "../src/evidence.js"; const android = { name: "android", @@ -422,6 +422,64 @@ test("composeAfterTest reports no fictitious paths when capture fails and preser assert.equal(consumerResult, result); }); +test("source-read rejection emits no complete evidence record and preserves the failed result", async () => { + const events: string[] = []; + const sourceError = new Error("page source unavailable"); + const captureFailure = async (prefix: string) => + captureStatePaths(prefix, { + async getPageSource() { + events.push("source"); + throw sourceError; + }, + async captureScreenshot(filename) { + events.push(`screenshot:${filename}`); + return `/artifacts/${filename}`; + }, + async captureText(filename) { + events.push(`source-file:${filename}`); + return `/artifacts/${filename}`; + }, + }); + const runnerTest = { + type: "test", + title: "fails", + parent: "suite", + fullTitle: "suite fails", + fullName: "suite fails", + pending: false, + file: "/proj/source-failure.spec.ts", + ctx: {}, + }; + const result = { + passed: false, + duration: 1, + retries: { limit: 0, attempts: 0 }, + exception: "original failure", + status: "failed", + }; + let consumerResult: unknown; + const hook = composeAfterTest( + async (_test, _context, receivedResult) => { + events.push("consumer"); + consumerResult = receivedResult; + }, + captureFailure, + { + project: "ios", + onFailureEvidence() { + events.push("evidence"); + }, + }, + ); + + await assert.rejects(() => captureFailure("direct-source-failure"), sourceError); + events.length = 0; + await hook(runnerTest, {}, result); + + assert.deepEqual(events, ["source", "consumer"]); + assert.equal(consumerResult, result); +}); + test("buildWdioConfig lets nativeproof.config.ts own the artifact directory", async () => { const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-artifacts-")); const configuredDir = path.join(dir, "configured"); @@ -459,7 +517,7 @@ test("buildWdioConfig adds a built-in evidence-on-failure afterTest hook", async test("failureEvidenceName is safe, capped and unique across file, project and retry identity", () => { assert.match( failureEvidenceName({ parent: "Room · onboarding", title: "Accept is inert!" }), - /^failure-Room_onboarding-Accept_is_inert_-[a-f0-9]{8}$/, + /^failure-Room_onboarding-Accept_is_inert_-[a-f0-9]{64}$/, ); const first = failureEvidenceName({ parent: "x".repeat(200), title: "one" }); const second = failureEvidenceName({ parent: "x".repeat(200), title: "two" }); @@ -484,6 +542,21 @@ test("failureEvidenceName is safe, capped and unique across file, project and re assert.equal(new Set(identities.map(failureEvidenceName)).size, identities.length); }); +test("failureEvidenceName distinguishes the exact known 32-bit digest collision", () => { + const identity = { + parent: "Login", + title: "rejects bad password", + fullName: "Login rejects bad password", + project: "android", + attempt: 0, + }; + const knownCollision = [ + { ...identity, file: "/proj/tests/case-64394.spec.ts" }, + { ...identity, file: "/proj/tests/case-84739.spec.ts" }, + ].map(failureEvidenceName); + assert.notEqual(knownCollision[0], knownCollision[1]); +}); + test("findConfigFile locates nativeproof.config.* via the injected exists check", () => { const exists = (file: string) => file.endsWith("nativeproof.config.ts"); assert.match(findConfigFile("/proj", exists) ?? "", /\/proj\/nativeproof\.config\.ts$/); From 2fa69fd26ebfe5c97f276774e68309a66c941788 Mon Sep 17 00:00:00 2001 From: Ben Sheridan-Edwards Date: Tue, 15 Sep 2026 16:17:03 +0100 Subject: [PATCH 3/4] fix(evidence): preserve public capture fallback --- CHANGELOG.md | 4 +++ src/evidence.ts | 25 +++++++++++++---- test/config.test.ts | 68 ++++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 91 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d5149db..d8a4696 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,10 @@ and the runner-native, Playwright-feeling test surface. aborting polling on the first thrown source/read. - Failure evidence filenames include a stable suffix, preventing truncated failing-test names from overwriting each other's `.png` / `.xml` artifacts. +- Public `captureState(prefix)` keeps its best-effort source-read fallback: it warns, + writes an empty XML snapshot alongside the screenshot, and resolves with `""`. + Internal `captureStatePaths` remains strict so source-read failures cannot emit a + misleading complete `onFailureEvidence` record. - `nativeproof inspect` skips transient input values when suggesting `getByText(...)` locators, so typed emails/search queries do not become brittle first-read selectors. diff --git a/src/evidence.ts b/src/evidence.ts index f44548b..f5a98f7 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -82,6 +82,19 @@ const defaultCaptureStateDependencies: CaptureStateDependencies = { captureText, }; +async function captureStateWithDependencies( + prefix: string, + dependencies: CaptureStateDependencies, + onSourceFailure?: (error: unknown) => string, +): Promise { + const source = onSourceFailure + ? await dependencies.getPageSource().catch(onSourceFailure) + : await dependencies.getPageSource(); + const pngPath = await dependencies.captureScreenshot(`${prefix}.png`); + const xmlPath = await dependencies.captureText(`${prefix}.xml`, source); + return { source, pngPath, xmlPath }; +} + /** @internal Capture a screenshot + redacted source pair and return only paths written successfully. */ export async function captureStatePaths( prefix: string, @@ -90,15 +103,17 @@ export async function captureStatePaths( // Reject the whole pair when source capture fails: an empty XML file would make a partial // capture look complete to structured evidence consumers. composeAfterTest keeps this // best-effort and preserves the original test failure + consumer hook invocation. - const source = await dependencies.getPageSource(); - const pngPath = await dependencies.captureScreenshot(`${prefix}.png`); - const xmlPath = await dependencies.captureText(`${prefix}.xml`, source); - return { source, pngPath, xmlPath }; + return captureStateWithDependencies(prefix, dependencies); } /** Capture a screenshot + redacted source pair under one prefix; returns the source. */ export async function captureState(prefix: string): Promise { - return (await captureStatePaths(prefix)).source; + return ( + await captureStateWithDependencies(prefix, defaultCaptureStateDependencies, (error) => { + console.warn(`[nativeproof] getPageSource failed during captureState("${prefix}"): ${error}`); + return ""; + }) + ).source; } /** diff --git a/test/config.test.ts b/test/config.test.ts index 473c385..ce691f9 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -3,6 +3,7 @@ import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "no import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; +import { _setGlobal } from "@wdio/globals"; import type { App, ScreenFactories } from "../src/app.js"; import { bootedIosSimulatorFromSimctl, @@ -13,7 +14,13 @@ import { resolveProject, splitSpecGlobs, } from "../src/config.js"; -import { captureStatePaths, captureText, failureEvidenceName, setArtifactDir } from "../src/evidence.js"; +import { + captureState, + captureStatePaths, + captureText, + failureEvidenceName, + setArtifactDir, +} from "../src/evidence.js"; const android = { name: "android", @@ -480,6 +487,65 @@ test("source-read rejection emits no complete evidence record and preserves the assert.equal(consumerResult, result); }); +test("public captureState warns and writes best-effort evidence when source capture fails", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-public-capture-state-")); + const warnings: unknown[][] = []; + const previousWarn = console.warn; + try { + setArtifactDir(dir); + _setGlobal("browser", { + async getPageSource() { + throw new Error("page source unavailable"); + }, + async saveScreenshot(target: string) { + writeFileSync(target, "png"); + }, + }); + console.warn = (...args: unknown[]) => warnings.push(args); + + assert.equal(await captureState("legacy-source-failure"), ""); + assert.equal(readFileSync(path.join(dir, "legacy-source-failure.png"), "utf8"), "png"); + assert.equal(readFileSync(path.join(dir, "legacy-source-failure.xml"), "utf8"), ""); + assert.deepEqual(warnings, [ + [ + '[nativeproof] getPageSource failed during captureState("legacy-source-failure"): Error: page source unavailable', + ], + ]); + } finally { + console.warn = previousWarn; + _setGlobal("browser", undefined); + setArtifactDir(undefined); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("public captureState does not misclassify screenshot failures as source failures", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-public-capture-state-")); + const screenshotError = new Error("screenshot unavailable"); + const warnings: unknown[][] = []; + const previousWarn = console.warn; + try { + setArtifactDir(dir); + _setGlobal("browser", { + async getPageSource() { + return ""; + }, + async saveScreenshot() { + throw screenshotError; + }, + }); + console.warn = (...args: unknown[]) => warnings.push(args); + + await assert.rejects(() => captureState("screenshot-failure"), screenshotError); + assert.deepEqual(warnings, []); + } finally { + console.warn = previousWarn; + _setGlobal("browser", undefined); + setArtifactDir(undefined); + rmSync(dir, { recursive: true, force: true }); + } +}); + test("buildWdioConfig lets nativeproof.config.ts own the artifact directory", async () => { const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-artifacts-")); const configuredDir = path.join(dir, "configured"); From 762961fe50eeb50e9bf14a1366725533eff298e4 Mon Sep 17 00:00:00 2001 From: Ben Sheridan-Edwards Date: Fri, 9 Oct 2026 14:03:02 +0100 Subject: [PATCH 4/4] fix(evidence): keep the failure screenshot when page source is unavailable The built-in failure hook used the strict capture, which rejects before the screenshot when getPageSource fails, and the rejection was swallowed. Every user lost the PNG and the warning that main still gives. The hook now falls back to the best-effort captureState in that case and skips the onFailureEvidence record, since only a complete pair should be reported. Also move the changelog entry out of the released 1.0.0 section into Unreleased, and describe the new callback and the filename change. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 21 +++++++++++--- src/config.ts | 22 ++++++++++++--- test/config.test.ts | 67 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 100 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8a4696..d533948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to NativeProof are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/) and the project adheres to [Semantic Versioning](https://semver.org/). +## Unreleased + +**Added** + +- `onFailureEvidence` in `nativeproof.config.ts` receives the project, spec file, + full test name, retry attempt and the exact `.png` / `.xml` paths each time the + built-in failure capture saves both files. When the page source cannot be read, + the callback is skipped and the built-in capture still saves the screenshot with + an empty XML snapshot and a warning. + +**Changed** + +- Failure evidence filenames now end in a full SHA-256 of the project, spec file, + full test name and retry attempt instead of an 8-character hash of the titles, so + retries and same-named tests in other files or projects no longer overwrite each + other. Names stay capped at 120 characters, so less of a long title survives. + ## 1.0.0 NativeProof's first stable release locks in the one-command init/onboard flow @@ -32,10 +49,6 @@ and the runner-native, Playwright-feeling test surface. aborting polling on the first thrown source/read. - Failure evidence filenames include a stable suffix, preventing truncated failing-test names from overwriting each other's `.png` / `.xml` artifacts. -- Public `captureState(prefix)` keeps its best-effort source-read fallback: it warns, - writes an empty XML snapshot alongside the screenshot, and resolves with `""`. - Internal `captureStatePaths` remains strict so source-read failures cannot emit a - misleading complete `onFailureEvidence` record. - `nativeproof inspect` skips transient input values when suggesting `getByText(...)` locators, so typed emails/search queries do not become brittle first-read selectors. diff --git a/src/config.ts b/src/config.ts index 3a6b33c..5246d17 100644 --- a/src/config.ts +++ b/src/config.ts @@ -5,6 +5,7 @@ import type { Frameworks, Reporters, Services } from "@wdio/types"; import type { App } from "./app.js"; import { type CapturedStatePaths, + captureState, captureStatePaths, failureEvidenceName, setArtifactDir, @@ -324,12 +325,14 @@ export function projectCapabilities(config: RunnerConfig, project: DeviceProject } type FailureCapture = (prefix: string) => Promise; +type FallbackCapture = (prefix: string) => Promise; /** @internal Compose public hooks without allowing them to replace default evidence capture. */ export function composeAfterTest( consumerHook?: RunnerAfterTestHook, captureFailure: FailureCapture = captureStatePaths, evidence: { project?: string; onFailureEvidence?: FailureEvidenceCallback } = {}, + fallbackCapture: FallbackCapture = captureState, ): RunnerAfterTestHook { return async (test, context, result) => { if (!result.passed) { @@ -342,8 +345,17 @@ export function composeAfterTest( parent: test.parent, title: test.title, }; - await captureFailure(failureEvidenceName(identity)) - .then(async (paths) => { + const prefix = failureEvidenceName(identity); + let paths: CapturedStatePaths | undefined; + try { + paths = await captureFailure(prefix); + } catch { + // The strict capture rejects a partial pair so no misleading record is reported. Fall + // back to best-effort captureState so the failure still leaves a screenshot and a warning. + await fallbackCapture(prefix).catch(() => {}); + } + if (paths) { + try { await evidence.onFailureEvidence?.({ project: identity.project, file: identity.file, @@ -352,8 +364,10 @@ export function composeAfterTest( pngPath: paths.pngPath, xmlPath: paths.xmlPath, }); - }) - .catch(() => {}); + } catch { + // A consumer's evidence callback must never mask the original test failure. + } + } } await consumerHook?.(test, context, result); }; diff --git a/test/config.test.ts b/test/config.test.ts index ce691f9..12b0b16 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -422,10 +422,13 @@ test("composeAfterTest reports no fictitious paths when capture fails and preser events.push("evidence"); }, }, + async () => { + events.push("fallback"); + }, ); await hook(runnerTest, {}, result); - assert.deepEqual(events, ["capture", "consumer"]); + assert.deepEqual(events, ["capture", "fallback", "consumer"]); assert.equal(consumerResult, result); }); @@ -477,16 +480,76 @@ test("source-read rejection emits no complete evidence record and preserves the events.push("evidence"); }, }, + async (prefix) => { + events.push( + `fallback:${prefix === failureEvidenceName({ project: "ios", file: runnerTest.file, fullName: runnerTest.fullName, attempt: 0, parent: "suite", title: "fails" })}`, + ); + }, ); await assert.rejects(() => captureFailure("direct-source-failure"), sourceError); events.length = 0; await hook(runnerTest, {}, result); - assert.deepEqual(events, ["source", "consumer"]); + assert.deepEqual(events, ["source", "fallback:true", "consumer"]); assert.equal(consumerResult, result); }); +test("built-in failure hook still saves a screenshot and warns when the page source is unavailable", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-default-failure-capture-")); + const warnings: unknown[][] = []; + const records: unknown[] = []; + const previousWarn = console.warn; + try { + _setGlobal("browser", { + async getPageSource() { + throw new Error("page source unavailable"); + }, + async saveScreenshot(target: string) { + writeFileSync(target, "png"); + }, + }); + console.warn = (...args: unknown[]) => warnings.push(args); + const wdio = buildWdioConfig( + { + projects, + artifacts: { dir }, + onFailureEvidence(record) { + records.push(record); + }, + }, + { platform: "ios" }, + "/proj", + ); + const afterTest = wdio.afterTest as (t: unknown, c: unknown, r: unknown) => Promise; + + await afterTest( + { title: "fails", parent: "suite", file: "/proj/tests/a.spec.ts", fullName: "suite fails" }, + {}, + { passed: false, retries: { attempts: 0 } }, + ); + + const prefix = failureEvidenceName({ + project: "ios", + file: "/proj/tests/a.spec.ts", + fullName: "suite fails", + attempt: 0, + parent: "suite", + title: "fails", + }); + assert.equal(readFileSync(path.join(dir, `${prefix}.png`), "utf8"), "png"); + assert.equal(readFileSync(path.join(dir, `${prefix}.xml`), "utf8"), ""); + assert.equal(warnings.length, 1); + assert.match(String(warnings[0]?.[0]), /getPageSource failed during captureState/); + assert.deepEqual(records, [], "a partial capture emits no complete evidence record"); + } finally { + console.warn = previousWarn; + _setGlobal("browser", undefined); + setArtifactDir(undefined); + rmSync(dir, { recursive: true, force: true }); + } +}); + test("public captureState warns and writes best-effort evidence when source capture fails", async () => { const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-public-capture-state-")); const warnings: unknown[][] = [];