diff --git a/CHANGELOG.md b/CHANGELOG.md index d5149db..d533948 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to NativeProof are documented here. The format follows [Keep a Changelog](https://keepachangelog.com/) and the project adheres to [Semantic Versioning](https://semver.org/). +## Unreleased + +**Added** + +- `onFailureEvidence` in `nativeproof.config.ts` receives the project, spec file, + full test name, retry attempt and the exact `.png` / `.xml` paths each time the + built-in failure capture saves both files. When the page source cannot be read, + the callback is skipped and the built-in capture still saves the screenshot with + an empty XML snapshot and a warning. + +**Changed** + +- Failure evidence filenames now end in a full SHA-256 of the project, spec file, + full test name and retry attempt instead of an 8-character hash of the titles, so + retries and same-named tests in other files or projects no longer overwrite each + other. Names stay capped at 120 characters, so less of a long title survives. + ## 1.0.0 NativeProof's first stable release locks in the one-command init/onboard flow diff --git a/README.md b/README.md index 2c02094..cbac04d 100644 --- a/README.md +++ b/README.md @@ -332,6 +332,10 @@ export default defineConfig({ }, specFileRetries: 2, // retry a failed spec file reporters: ["spec", ["junit", { outputDir: "reports" }]], + async onFailureEvidence({ project, file, fullName, attempt, pngPath, xmlPath }) { + // Attach or index the exact files NativeProof successfully saved for this attempt. + console.log({ project, file, fullName, attempt, pngPath, xmlPath }); + }, async afterTest(test, _context, result) { // NativeProof still captures its default failure evidence before this hook runs. console.log(`${test.parent} > ${test.title}: ${result.passed ? "passed" : "failed"}`); diff --git a/src/config.ts b/src/config.ts index 16b85fc..5246d17 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,7 +3,13 @@ import { existsSync } from "node:fs"; import path from "node:path"; import type { Frameworks, Reporters, Services } from "@wdio/types"; import type { App } from "./app.js"; -import { captureState, failureEvidenceName, setArtifactDir } from "./evidence.js"; +import { + type CapturedStatePaths, + captureState, + captureStatePaths, + failureEvidenceName, + setArtifactDir, +} from "./evidence.js"; /** * The Playwright-style config: one `nativeproof.config.ts` declares the app, the device @@ -167,6 +173,19 @@ export type RunnerTest = Frameworks.Test; /** WebdriverIO's complete runner result payload, including retries, duration, status, and error. */ export type RunnerTestResult = Frameworks.TestResult; +/** Exact files produced by NativeProof's built-in capture for one failed test attempt. */ +export interface FailureEvidence { + project: string; + file: string; + fullName: string; + attempt: number; + pngPath: string; + xmlPath: string; +} + +/** Reporter-neutral notification after built-in failure evidence is saved successfully. */ +export type FailureEvidenceCallback = (evidence: FailureEvidence) => unknown | Promise; + /** Consumer hook composed after NativeProof's built-in failure evidence capture. */ export type RunnerAfterTestHook = NonNullable; @@ -196,6 +215,8 @@ export interface RunnerConfig { reporters?: RunnerReporter[]; /** Consumer lifecycle hook; failure capture still runs first. */ afterTest?: RunnerAfterTestHook; + /** Called with the exact built-in PNG/XML files after a failed test capture succeeds. */ + onFailureEvidence?: FailureEvidenceCallback; /** * WebdriverIO pass-throughs for tuning real-device runs. Each is forwarded only when set, so * WebdriverIO's own defaults apply otherwise. Slow software-GPU emulators in particular often @@ -303,16 +324,50 @@ export function projectCapabilities(config: RunnerConfig, project: DeviceProject }; } -type FailureCapture = (prefix: string) => Promise; +type FailureCapture = (prefix: string) => Promise; +type FallbackCapture = (prefix: string) => Promise; /** @internal Compose public hooks without allowing them to replace default evidence capture. */ export function composeAfterTest( consumerHook?: RunnerAfterTestHook, - captureFailure: FailureCapture = captureState, + captureFailure: FailureCapture = captureStatePaths, + evidence: { project?: string; onFailureEvidence?: FailureEvidenceCallback } = {}, + fallbackCapture: FallbackCapture = captureState, ): RunnerAfterTestHook { return async (test, context, result) => { if (!result.passed) { - await captureFailure(failureEvidenceName(test)).catch(() => {}); + const attempt = result.retries?.attempts ?? 0; + const identity = { + project: evidence.project ?? "", + file: test.file ?? "", + fullName: test.fullName ?? `${test.parent} ${test.title}`, + attempt, + parent: test.parent, + title: test.title, + }; + const prefix = failureEvidenceName(identity); + let paths: CapturedStatePaths | undefined; + try { + paths = await captureFailure(prefix); + } catch { + // The strict capture rejects a partial pair so no misleading record is reported. Fall + // back to best-effort captureState so the failure still leaves a screenshot and a warning. + await fallbackCapture(prefix).catch(() => {}); + } + if (paths) { + try { + await evidence.onFailureEvidence?.({ + project: identity.project, + file: identity.file, + fullName: identity.fullName, + attempt, + pngPath: paths.pngPath, + xmlPath: paths.xmlPath, + }); + } catch { + // A consumer's evidence callback must never mask the original test failure. + } + } } await consumerHook?.(test, context, result); }; @@ -353,7 +408,10 @@ export function buildWdioConfig( // redacted page source into the artifact dir, named after the spec. Best-effort — a // capture error never masks the real failure — and consumers get it without writing // their own afterTest hook. - afterTest: composeAfterTest(config.afterTest), + afterTest: composeAfterTest(config.afterTest, captureStatePaths, { + project: project.name, + ...(config.onFailureEvidence ? { onFailureEvidence: config.onFailureEvidence } : {}), + }), }; // Optional WebdriverIO tuning — forwarded only when the consumer set it, so wdio's defaults apply // otherwise (real emulators/simulators often need longer connection/wait timeouts than the defaults). diff --git a/src/evidence.ts b/src/evidence.ts index 3edc480..f5a98f7 100644 --- a/src/evidence.ts +++ b/src/evidence.ts @@ -65,26 +65,80 @@ export async function captureScreenshot(filename: string): Promise { return target; } +export interface CapturedStatePaths { + pngPath: string; + xmlPath: string; +} + +interface CaptureStateDependencies { + getPageSource(): Promise; + captureScreenshot(filename: string): Promise; + captureText(filename: string, contents: string): Promise; +} + +const defaultCaptureStateDependencies: CaptureStateDependencies = { + getPageSource: () => browser.getPageSource(), + captureScreenshot, + captureText, +}; + +async function captureStateWithDependencies( + prefix: string, + dependencies: CaptureStateDependencies, + onSourceFailure?: (error: unknown) => string, +): Promise { + const source = onSourceFailure + ? await dependencies.getPageSource().catch(onSourceFailure) + : await dependencies.getPageSource(); + const pngPath = await dependencies.captureScreenshot(`${prefix}.png`); + const xmlPath = await dependencies.captureText(`${prefix}.xml`, source); + return { source, pngPath, xmlPath }; +} + +/** @internal Capture a screenshot + redacted source pair and return only paths written successfully. */ +export async function captureStatePaths( + prefix: string, + dependencies: CaptureStateDependencies = defaultCaptureStateDependencies, +): Promise { + // Reject the whole pair when source capture fails: an empty XML file would make a partial + // capture look complete to structured evidence consumers. composeAfterTest keeps this + // best-effort and preserves the original test failure + consumer hook invocation. + return captureStateWithDependencies(prefix, dependencies); +} + /** Capture a screenshot + redacted source pair under one prefix; returns the source. */ export async function captureState(prefix: string): Promise { - const source = await browser.getPageSource().catch((err: unknown) => { - // A failed capture must not look like a clean empty screen in the evidence trail. - console.warn(`[nativeproof] getPageSource failed during captureState("${prefix}"): ${err}`); - return ""; - }); - await captureScreenshot(`${prefix}.png`); - await captureText(`${prefix}.xml`, source); - return source; + return ( + await captureStateWithDependencies(prefix, defaultCaptureStateDependencies, (error) => { + console.warn(`[nativeproof] getPageSource failed during captureState("${prefix}"): ${error}`); + return ""; + }) + ).source; } /** * A filesystem-safe evidence prefix for a failed behaviour — `failure--` - * with runs of non-word characters collapsed to `_`, a short stable suffix to avoid + * with runs of non-word characters collapsed to `_`, a full stable digest to avoid * truncation collisions, and capped at 120 chars. Used by the runner's built-in on-failure * capture so a failing spec leaves a screenshot + source pair with no per-spec wiring. */ -export function failureEvidenceName(test: { parent: string; title: string }): string { +export function failureEvidenceName(test: { + parent: string; + title: string; + project?: string; + file?: string; + fullName?: string; + attempt?: number; +}): string { const raw = `failure-${test.parent}-${test.title}`; - const suffix = `-${createHash("sha1").update(raw).digest("hex").slice(0, 8)}`; + const identity = [ + test.project ?? "", + test.file ?? "", + test.fullName ?? "", + String(test.attempt ?? 0), + test.parent, + test.title, + ].join("\0"); + const suffix = `-${createHash("sha256").update(identity).digest("hex")}`; return `${raw.replace(/[^\w.-]+/g, "_").slice(0, 120 - suffix.length)}${suffix}`; } diff --git a/src/index.ts b/src/index.ts index 99a5efa..a970714 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,6 +20,8 @@ export { defineApp } from "./app.js"; export type { AppiumOptions, DeviceProject, + FailureEvidence, + FailureEvidenceCallback, NativeProofConfig, RunnerAfterTestHook, RunnerConfig, diff --git a/test/config.test.ts b/test/config.test.ts index 69937e0..12b0b16 100644 --- a/test/config.test.ts +++ b/test/config.test.ts @@ -1,8 +1,9 @@ import assert from "node:assert/strict"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; +import { _setGlobal } from "@wdio/globals"; import type { App, ScreenFactories } from "../src/app.js"; import { bootedIosSimulatorFromSimctl, @@ -13,7 +14,13 @@ import { resolveProject, splitSpecGlobs, } from "../src/config.js"; -import { captureText, failureEvidenceName, setArtifactDir } from "../src/evidence.js"; +import { + captureState, + captureStatePaths, + captureText, + failureEvidenceName, + setArtifactDir, +} from "../src/evidence.js"; const android = { name: "android", @@ -300,7 +307,8 @@ test("an explicit empty CLI grep clears config grep without replacing other Moch }); }); -test("composeAfterTest captures failures before the consumer hook and keeps capture best-effort", async () => { +test("composeAfterTest reports exact saved failure evidence before the consumer hook", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-failure-evidence-")); const events: string[] = []; const runnerTest = { type: "test", @@ -309,45 +317,296 @@ test("composeAfterTest captures failures before the consumer hook and keeps capt fullTitle: "suite fails", fullName: "suite fails", pending: false, - file: "tests/failure.spec.ts", + file: "/proj/tests/login.spec.ts", ctx: {}, }; + try { + const pngPath = path.join(dir, "failure.png"); + const xmlPath = path.join(dir, "failure.xml"); + const records: unknown[] = []; + const hook = composeAfterTest( + async () => { + events.push("consumer"); + }, + async () => { + events.push("capture"); + writeFileSync(pngPath, "png"); + writeFileSync(xmlPath, ""); + return { pngPath, xmlPath }; + }, + { + project: "ios", + onFailureEvidence(record) { + events.push("evidence"); + records.push(record); + }, + }, + ); + + await hook( + runnerTest, + {}, + { + passed: false, + duration: 1, + retries: { limit: 1, attempts: 1 }, + exception: "failure", + status: "failed", + }, + ); + assert.deepEqual(events, ["capture", "evidence", "consumer"]); + assert.deepEqual(records, [ + { + project: "ios", + file: "/proj/tests/login.spec.ts", + fullName: "suite fails", + attempt: 1, + pngPath, + xmlPath, + }, + ]); + assert.equal(existsSync(pngPath), true); + assert.equal(existsSync(xmlPath), true); + + events.length = 0; + await hook( + { ...runnerTest, title: "passes", fullTitle: "suite passes", fullName: "suite passes" }, + {}, + { + passed: true, + duration: 1, + retries: { limit: 1, attempts: 0 }, + exception: "", + status: "passed", + }, + ); + assert.deepEqual(events, ["consumer"]); + assert.equal(records.length, 1, "passing tests emit no failure-evidence callback"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("composeAfterTest reports no fictitious paths when capture fails and preserves the failure", async () => { + const events: string[] = []; + const runnerTest = { + type: "test", + title: "fails", + parent: "suite", + fullTitle: "suite fails", + fullName: "suite fails", + pending: false, + file: "/proj/fail.spec.ts", + ctx: {}, + }; + const result = { + passed: false, + duration: 1, + retries: { limit: 0, attempts: 0 }, + exception: "failure", + status: "failed", + }; + let consumerResult: unknown; const hook = composeAfterTest( - async () => { + async (_test, _context, receivedResult) => { events.push("consumer"); + consumerResult = receivedResult; }, async () => { events.push("capture"); throw new Error("device disconnected during evidence capture"); }, - ); - - await hook( - runnerTest, - {}, { - passed: false, - duration: 1, - retries: { limit: 0, attempts: 0 }, - exception: "failure", - status: "failed", + project: "android", + onFailureEvidence() { + events.push("evidence"); + }, + }, + async () => { + events.push("fallback"); }, ); - assert.deepEqual(events, ["capture", "consumer"]); - events.length = 0; - await hook( - { ...runnerTest, title: "passes", fullTitle: "suite passes", fullName: "suite passes" }, - {}, + await hook(runnerTest, {}, result); + assert.deepEqual(events, ["capture", "fallback", "consumer"]); + assert.equal(consumerResult, result); +}); + +test("source-read rejection emits no complete evidence record and preserves the failed result", async () => { + const events: string[] = []; + const sourceError = new Error("page source unavailable"); + const captureFailure = async (prefix: string) => + captureStatePaths(prefix, { + async getPageSource() { + events.push("source"); + throw sourceError; + }, + async captureScreenshot(filename) { + events.push(`screenshot:${filename}`); + return `/artifacts/${filename}`; + }, + async captureText(filename) { + events.push(`source-file:${filename}`); + return `/artifacts/${filename}`; + }, + }); + const runnerTest = { + type: "test", + title: "fails", + parent: "suite", + fullTitle: "suite fails", + fullName: "suite fails", + pending: false, + file: "/proj/source-failure.spec.ts", + ctx: {}, + }; + const result = { + passed: false, + duration: 1, + retries: { limit: 0, attempts: 0 }, + exception: "original failure", + status: "failed", + }; + let consumerResult: unknown; + const hook = composeAfterTest( + async (_test, _context, receivedResult) => { + events.push("consumer"); + consumerResult = receivedResult; + }, + captureFailure, { - passed: true, - duration: 1, - retries: { limit: 0, attempts: 0 }, - exception: "", - status: "passed", + project: "ios", + onFailureEvidence() { + events.push("evidence"); + }, + }, + async (prefix) => { + events.push( + `fallback:${prefix === failureEvidenceName({ project: "ios", file: runnerTest.file, fullName: runnerTest.fullName, attempt: 0, parent: "suite", title: "fails" })}`, + ); }, ); - assert.deepEqual(events, ["consumer"]); + + await assert.rejects(() => captureFailure("direct-source-failure"), sourceError); + events.length = 0; + await hook(runnerTest, {}, result); + + assert.deepEqual(events, ["source", "fallback:true", "consumer"]); + assert.equal(consumerResult, result); +}); + +test("built-in failure hook still saves a screenshot and warns when the page source is unavailable", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-default-failure-capture-")); + const warnings: unknown[][] = []; + const records: unknown[] = []; + const previousWarn = console.warn; + try { + _setGlobal("browser", { + async getPageSource() { + throw new Error("page source unavailable"); + }, + async saveScreenshot(target: string) { + writeFileSync(target, "png"); + }, + }); + console.warn = (...args: unknown[]) => warnings.push(args); + const wdio = buildWdioConfig( + { + projects, + artifacts: { dir }, + onFailureEvidence(record) { + records.push(record); + }, + }, + { platform: "ios" }, + "/proj", + ); + const afterTest = wdio.afterTest as (t: unknown, c: unknown, r: unknown) => Promise; + + await afterTest( + { title: "fails", parent: "suite", file: "/proj/tests/a.spec.ts", fullName: "suite fails" }, + {}, + { passed: false, retries: { attempts: 0 } }, + ); + + const prefix = failureEvidenceName({ + project: "ios", + file: "/proj/tests/a.spec.ts", + fullName: "suite fails", + attempt: 0, + parent: "suite", + title: "fails", + }); + assert.equal(readFileSync(path.join(dir, `${prefix}.png`), "utf8"), "png"); + assert.equal(readFileSync(path.join(dir, `${prefix}.xml`), "utf8"), ""); + assert.equal(warnings.length, 1); + assert.match(String(warnings[0]?.[0]), /getPageSource failed during captureState/); + assert.deepEqual(records, [], "a partial capture emits no complete evidence record"); + } finally { + console.warn = previousWarn; + _setGlobal("browser", undefined); + setArtifactDir(undefined); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("public captureState warns and writes best-effort evidence when source capture fails", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-public-capture-state-")); + const warnings: unknown[][] = []; + const previousWarn = console.warn; + try { + setArtifactDir(dir); + _setGlobal("browser", { + async getPageSource() { + throw new Error("page source unavailable"); + }, + async saveScreenshot(target: string) { + writeFileSync(target, "png"); + }, + }); + console.warn = (...args: unknown[]) => warnings.push(args); + + assert.equal(await captureState("legacy-source-failure"), ""); + assert.equal(readFileSync(path.join(dir, "legacy-source-failure.png"), "utf8"), "png"); + assert.equal(readFileSync(path.join(dir, "legacy-source-failure.xml"), "utf8"), ""); + assert.deepEqual(warnings, [ + [ + '[nativeproof] getPageSource failed during captureState("legacy-source-failure"): Error: page source unavailable', + ], + ]); + } finally { + console.warn = previousWarn; + _setGlobal("browser", undefined); + setArtifactDir(undefined); + rmSync(dir, { recursive: true, force: true }); + } +}); + +test("public captureState does not misclassify screenshot failures as source failures", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "nativeproof-public-capture-state-")); + const screenshotError = new Error("screenshot unavailable"); + const warnings: unknown[][] = []; + const previousWarn = console.warn; + try { + setArtifactDir(dir); + _setGlobal("browser", { + async getPageSource() { + return ""; + }, + async saveScreenshot() { + throw screenshotError; + }, + }); + console.warn = (...args: unknown[]) => warnings.push(args); + + await assert.rejects(() => captureState("screenshot-failure"), screenshotError); + assert.deepEqual(warnings, []); + } finally { + console.warn = previousWarn; + _setGlobal("browser", undefined); + setArtifactDir(undefined); + rmSync(dir, { recursive: true, force: true }); + } }); test("buildWdioConfig lets nativeproof.config.ts own the artifact directory", async () => { @@ -384,16 +643,47 @@ test("buildWdioConfig adds a built-in evidence-on-failure afterTest hook", async await afterTest({ title: "t", parent: "p" }, {}, { passed: false }); }); -test("failureEvidenceName builds a filesystem-safe, capped and unique prefix", () => { +test("failureEvidenceName is safe, capped and unique across file, project and retry identity", () => { assert.match( failureEvidenceName({ parent: "Room · onboarding", title: "Accept is inert!" }), - /^failure-Room_onboarding-Accept_is_inert_-[a-f0-9]{8}$/, + /^failure-Room_onboarding-Accept_is_inert_-[a-f0-9]{64}$/, ); const first = failureEvidenceName({ parent: "x".repeat(200), title: "one" }); const second = failureEvidenceName({ parent: "x".repeat(200), title: "two" }); assert.ok(first.length <= 120); assert.ok(second.length <= 120); assert.notEqual(first, second); + + const testIdentity = { + parent: "Login", + title: "rejects bad password", + fullName: "Login rejects bad password", + file: "/proj/tests/login.spec.ts", + project: "android", + attempt: 0, + }; + const identities = [ + testIdentity, + { ...testIdentity, file: "/proj/tests/admin-login.spec.ts" }, + { ...testIdentity, project: "ios" }, + { ...testIdentity, attempt: 1 }, + ]; + assert.equal(new Set(identities.map(failureEvidenceName)).size, identities.length); +}); + +test("failureEvidenceName distinguishes the exact known 32-bit digest collision", () => { + const identity = { + parent: "Login", + title: "rejects bad password", + fullName: "Login rejects bad password", + project: "android", + attempt: 0, + }; + const knownCollision = [ + { ...identity, file: "/proj/tests/case-64394.spec.ts" }, + { ...identity, file: "/proj/tests/case-84739.spec.ts" }, + ].map(failureEvidenceName); + assert.notEqual(knownCollision[0], knownCollision[1]); }); test("findConfigFile locates nativeproof.config.* via the injected exists check", () => { diff --git a/test/index.test.ts b/test/index.test.ts index 195ae0f..58646d1 100644 --- a/test/index.test.ts +++ b/test/index.test.ts @@ -15,6 +15,8 @@ import { defineApp, defineConfig, expect, + type FailureEvidence, + type FailureEvidenceCallback, type FailureInfo, type FlowContext, type FrameDirection, @@ -60,6 +62,8 @@ type PublicTypeSmoke = { appium: AppiumOptions; device: DeviceContext; driver: Driver; + failureEvidence: FailureEvidence; + failureEvidenceCallback: FailureEvidenceCallback; failure: FailureInfo; flow: FlowContext; frameDirection: FrameDirection;