Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,23 @@ All notable changes to NativeProof are documented here. The format follows
[Keep a Changelog](https://keepachangelog.com/) and the project adheres to
[Semantic Versioning](https://semver.org/).

## Unreleased

**Added**

- `onFailureEvidence` in `nativeproof.config.ts` receives the project, spec file,
full test name, retry attempt and the exact `.png` / `.xml` paths each time the
built-in failure capture saves both files. When the page source cannot be read,
the callback is skipped and the built-in capture still saves the screenshot with
an empty XML snapshot and a warning.

**Changed**

- Failure evidence filenames now end in a full SHA-256 of the project, spec file,
full test name and retry attempt instead of an 8-character hash of the titles, so
retries and same-named tests in other files or projects no longer overwrite each
other. Names stay capped at 120 characters, so less of a long title survives.

## 1.0.0

NativeProof's first stable release locks in the one-command init/onboard flow
Expand Down
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,10 @@ export default defineConfig({
},
specFileRetries: 2, // retry a failed spec file
reporters: ["spec", ["junit", { outputDir: "reports" }]],
async onFailureEvidence({ project, file, fullName, attempt, pngPath, xmlPath }) {
// Attach or index the exact files NativeProof successfully saved for this attempt.
console.log({ project, file, fullName, attempt, pngPath, xmlPath });
},
async afterTest(test, _context, result) {
// NativeProof still captures its default failure evidence before this hook runs.
console.log(`${test.parent} > ${test.title}: ${result.passed ? "passed" : "failed"}`);
Expand Down
68 changes: 63 additions & 5 deletions src/config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,13 @@ import { existsSync } from "node:fs";
import path from "node:path";
import type { Frameworks, Reporters, Services } from "@wdio/types";
import type { App } from "./app.js";
import { captureState, failureEvidenceName, setArtifactDir } from "./evidence.js";
import {
type CapturedStatePaths,
captureState,
captureStatePaths,
failureEvidenceName,
setArtifactDir,
} from "./evidence.js";

/**
* The Playwright-style config: one `nativeproof.config.ts` declares the app, the device
Expand Down Expand Up @@ -167,6 +173,19 @@ export type RunnerTest = Frameworks.Test;
/** WebdriverIO's complete runner result payload, including retries, duration, status, and error. */
export type RunnerTestResult = Frameworks.TestResult;

/** Exact files produced by NativeProof's built-in capture for one failed test attempt. */
export interface FailureEvidence {
project: string;
file: string;
fullName: string;
attempt: number;
pngPath: string;
xmlPath: string;
}

/** Reporter-neutral notification after built-in failure evidence is saved successfully. */
export type FailureEvidenceCallback = (evidence: FailureEvidence) => unknown | Promise<unknown>;

/** Consumer hook composed after NativeProof's built-in failure evidence capture. */
export type RunnerAfterTestHook = NonNullable<Services.HookFunctions["afterTest"]>;

Expand Down Expand Up @@ -196,6 +215,8 @@ export interface RunnerConfig {
reporters?: RunnerReporter[];
/** Consumer lifecycle hook; failure capture still runs first. */
afterTest?: RunnerAfterTestHook;
/** Called with the exact built-in PNG/XML files after a failed test capture succeeds. */
onFailureEvidence?: FailureEvidenceCallback;
/**
* WebdriverIO pass-throughs for tuning real-device runs. Each is forwarded only when set, so
* WebdriverIO's own defaults apply otherwise. Slow software-GPU emulators in particular often
Expand Down Expand Up @@ -303,16 +324,50 @@ export function projectCapabilities(config: RunnerConfig, project: DeviceProject
};
}

type FailureCapture = (prefix: string) => Promise<unknown>;
type FailureCapture = (prefix: string) => Promise<CapturedStatePaths>;
type FallbackCapture = (prefix: string) => Promise<unknown>;

/** @internal Compose public hooks without allowing them to replace default evidence capture. */
export function composeAfterTest(
consumerHook?: RunnerAfterTestHook,
captureFailure: FailureCapture = captureState,
captureFailure: FailureCapture = captureStatePaths,
evidence: { project?: string; onFailureEvidence?: FailureEvidenceCallback } = {},
fallbackCapture: FallbackCapture = captureState,
): RunnerAfterTestHook {
return async (test, context, result) => {
if (!result.passed) {
await captureFailure(failureEvidenceName(test)).catch(() => {});
const attempt = result.retries?.attempts ?? 0;
const identity = {
project: evidence.project ?? "",
file: test.file ?? "",
fullName: test.fullName ?? `${test.parent} ${test.title}`,
attempt,
parent: test.parent,
title: test.title,
};
const prefix = failureEvidenceName(identity);
let paths: CapturedStatePaths | undefined;
try {
paths = await captureFailure(prefix);
} catch {
// The strict capture rejects a partial pair so no misleading record is reported. Fall
// back to best-effort captureState so the failure still leaves a screenshot and a warning.
await fallbackCapture(prefix).catch(() => {});
}
if (paths) {
try {
await evidence.onFailureEvidence?.({
project: identity.project,
file: identity.file,
fullName: identity.fullName,
attempt,
pngPath: paths.pngPath,
xmlPath: paths.xmlPath,
});
} catch {
// A consumer's evidence callback must never mask the original test failure.
}
}
}
await consumerHook?.(test, context, result);
};
Expand Down Expand Up @@ -353,7 +408,10 @@ export function buildWdioConfig(
// redacted page source into the artifact dir, named after the spec. Best-effort — a
// capture error never masks the real failure — and consumers get it without writing
// their own afterTest hook.
afterTest: composeAfterTest(config.afterTest),
afterTest: composeAfterTest(config.afterTest, captureStatePaths, {
project: project.name,
...(config.onFailureEvidence ? { onFailureEvidence: config.onFailureEvidence } : {}),
}),
};
// Optional WebdriverIO tuning — forwarded only when the consumer set it, so wdio's defaults apply
// otherwise (real emulators/simulators often need longer connection/wait timeouts than the defaults).
Expand Down
76 changes: 65 additions & 11 deletions src/evidence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -65,26 +65,80 @@ export async function captureScreenshot(filename: string): Promise<string> {
return target;
}

export interface CapturedStatePaths {
pngPath: string;
xmlPath: string;
}

interface CaptureStateDependencies {
getPageSource(): Promise<string>;
captureScreenshot(filename: string): Promise<string>;
captureText(filename: string, contents: string): Promise<string>;
}

const defaultCaptureStateDependencies: CaptureStateDependencies = {
getPageSource: () => browser.getPageSource(),
captureScreenshot,
captureText,
};

async function captureStateWithDependencies(
prefix: string,
dependencies: CaptureStateDependencies,
onSourceFailure?: (error: unknown) => string,
): Promise<CapturedStatePaths & { source: string }> {
const source = onSourceFailure
? await dependencies.getPageSource().catch(onSourceFailure)
: await dependencies.getPageSource();
const pngPath = await dependencies.captureScreenshot(`${prefix}.png`);
const xmlPath = await dependencies.captureText(`${prefix}.xml`, source);
return { source, pngPath, xmlPath };
}

/** @internal Capture a screenshot + redacted source pair and return only paths written successfully. */
export async function captureStatePaths(
prefix: string,
dependencies: CaptureStateDependencies = defaultCaptureStateDependencies,
): Promise<CapturedStatePaths & { source: string }> {
// Reject the whole pair when source capture fails: an empty XML file would make a partial
// capture look complete to structured evidence consumers. composeAfterTest keeps this
// best-effort and preserves the original test failure + consumer hook invocation.
return captureStateWithDependencies(prefix, dependencies);
}

/** Capture a screenshot + redacted source pair under one prefix; returns the source. */
export async function captureState(prefix: string): Promise<string> {
const source = await browser.getPageSource().catch((err: unknown) => {
// A failed capture must not look like a clean empty screen in the evidence trail.
console.warn(`[nativeproof] getPageSource failed during captureState("${prefix}"): ${err}`);
return "";
});
await captureScreenshot(`${prefix}.png`);
await captureText(`${prefix}.xml`, source);
return source;
return (
await captureStateWithDependencies(prefix, defaultCaptureStateDependencies, (error) => {
console.warn(`[nativeproof] getPageSource failed during captureState("${prefix}"): ${error}`);
return "";
})
).source;
}

/**
* A filesystem-safe evidence prefix for a failed behaviour — `failure-<describe>-<test>`
* with runs of non-word characters collapsed to `_`, a short stable suffix to avoid
* with runs of non-word characters collapsed to `_`, a full stable digest to avoid
* truncation collisions, and capped at 120 chars. Used by the runner's built-in on-failure
* capture so a failing spec leaves a screenshot + source pair with no per-spec wiring.
*/
export function failureEvidenceName(test: { parent: string; title: string }): string {
export function failureEvidenceName(test: {
parent: string;
title: string;
project?: string;
file?: string;
fullName?: string;
attempt?: number;
}): string {
const raw = `failure-${test.parent}-${test.title}`;
const suffix = `-${createHash("sha1").update(raw).digest("hex").slice(0, 8)}`;
const identity = [
test.project ?? "",
test.file ?? "",
test.fullName ?? "",
String(test.attempt ?? 0),
test.parent,
test.title,
].join("\0");
const suffix = `-${createHash("sha256").update(identity).digest("hex")}`;
return `${raw.replace(/[^\w.-]+/g, "_").slice(0, 120 - suffix.length)}${suffix}`;
}
2 changes: 2 additions & 0 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ export { defineApp } from "./app.js";
export type {
AppiumOptions,
DeviceProject,
FailureEvidence,
FailureEvidenceCallback,
NativeProofConfig,
RunnerAfterTestHook,
RunnerConfig,
Expand Down
Loading
Loading