Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -317,9 +317,31 @@ npx nativeproof --ios
npx nativeproof --android
npx nativeproof --project ios
npx nativeproof --spec tests/login.spec.ts
npx nativeproof --spec 'tests/{login,settings}.spec.ts' --grep '@smoke'
npx nativeproof --no-appium
```

Runner-native filtering, retries, and reporting stay in `nativeproof.config.ts`:

```ts
export default defineConfig({
projects: [/* device projects */],
mochaOpts: {
grep: "@smoke",
retries: 1, // retry a failed test
},
specFileRetries: 2, // retry a failed spec file
reporters: ["spec", ["junit", { outputDir: "reports" }]],
async afterTest(test, _context, result) {
// NativeProof still captures its default failure evidence before this hook runs.
console.log(`${test.parent} > ${test.title}: ${result.passed ? "passed" : "failed"}`);
},
});
```

`--spec` accepts comma-separated globs and brace expansions for selecting files. `--grep` overrides
only `mochaOpts.grep` for that invocation, so reporter and retry configuration remains unchanged.

Bin aliases:

```bash
Expand Down
1 change: 1 addition & 0 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
"@wdio/local-runner": "^9.29.1",
"@wdio/mocha-framework": "^9.29.1",
"@wdio/spec-reporter": "^9.29.1",
"@wdio/types": "^9.29.1",
"appium": "^3.5.2",
"tsx": "^4.22.4",
"webdriverio": "^9.21.1",
Expand Down
9 changes: 8 additions & 1 deletion src/cli.ts
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ const tsxLoader = pathToFileURL(packageRequire.resolve("tsx")).href;
*
* It resolves `nativeproof.config.ts`, ensures the configured Appium server is up (starting one
* if needed), and runs the suite with sane env (NATIVEPROOF_PLATFORM / NATIVEPROOF_SPEC /
* NATIVEPROOF_PROJECT) — so a consumer types one command instead of remembering env vars and a runner invocation. The
* NATIVEPROOF_PROJECT / NATIVEPROOF_GREP) — so a consumer types one command instead of remembering env vars and a runner invocation. The
* device/emulator itself is the environment (the mobile analogue of needing a display) and is left
* to the host.
*/
Expand All @@ -47,6 +47,7 @@ export interface CliArgs {
onboardPath: string | undefined;
project: string | undefined;
spec: string | undefined;
grep: string | undefined;
startAppium: boolean;
}

Expand All @@ -57,6 +58,7 @@ const DEFAULTS: CliArgs = {
onboardPath: undefined,
project: undefined,
spec: undefined,
grep: undefined,
startAppium: true,
};

Expand Down Expand Up @@ -116,6 +118,9 @@ export function parseArgs(
} else if (arg === "--spec") {
i += 1;
args.spec = valueFor(argv, i, "--spec");
} else if (arg === "--grep") {
i += 1;
args.grep = valueFor(argv, i, "--grep");
} else if (arg === "--platform") {
i += 1;
const platform = valueFor(argv, i, "--platform");
Expand Down Expand Up @@ -164,6 +169,7 @@ export function helpText(): string {
" --platform <android|ios> platform to run (sets NATIVEPROOF_PLATFORM)",
" --project <name> run a named project (sets NATIVEPROOF_PROJECT)",
" --spec <glob> run only matching specs (sets NATIVEPROOF_SPEC)",
" --grep <pattern> run tests matching Mocha grep (sets NATIVEPROOF_GREP)",
" --no-appium do not auto-start an Appium server",
" -h, --help show this help",
" -v, --version print the version",
Expand Down Expand Up @@ -1214,6 +1220,7 @@ export function runnerEnv(args: CliArgs, baseEnv: NodeJS.ProcessEnv = process.en
if (args.platform) env.NATIVEPROOF_PLATFORM = args.platform;
if (args.project) env.NATIVEPROOF_PROJECT = args.project;
if (args.spec) env.NATIVEPROOF_SPEC = args.spec;
if (args.grep !== undefined) env.NATIVEPROOF_GREP = args.grep;
return env;
}

Expand Down
84 changes: 74 additions & 10 deletions src/config.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { execFileSync } from "node:child_process";
import { existsSync } from "node:fs";
import path from "node:path";
import type { Frameworks, Reporters, Services } from "@wdio/types";
import type { App } from "./app.js";
import { captureState, failureEvidenceName, setArtifactDir } from "./evidence.js";

Expand Down Expand Up @@ -136,6 +137,39 @@ function hostDeviceDefaults(config: RunnerConfig, project: DeviceProject): Recor
: {};
}

/** Runner-native Mocha options forwarded without inventing a NativeProof test DSL. */
export interface RunnerMochaOptions {
require?: string[];
compilers?: string[];
allowUncaught?: boolean;
asyncOnly?: boolean;
bail?: boolean;
checkLeaks?: boolean;
delay?: boolean;
fgrep?: string;
forbidOnly?: boolean;
forbidPending?: boolean;
fullTrace?: boolean;
global?: string[];
grep?: RegExp | string;
invert?: boolean;
retries?: number;
timeout?: number | string;
ui?: "bdd" | "tdd" | "qunit" | "exports";
}

/** A built-in or installed WebdriverIO reporter, optionally with reporter-native options. */
export type RunnerReporter = Reporters.ReporterEntry;

/** WebdriverIO's complete runner test payload. */
export type RunnerTest = Frameworks.Test;

/** WebdriverIO's complete runner result payload, including retries, duration, status, and error. */
export type RunnerTestResult = Frameworks.TestResult;

/** Consumer hook composed after NativeProof's built-in failure evidence capture. */
export type RunnerAfterTestHook = NonNullable<Services.HookFunctions["afterTest"]>;

/** The device/run config the CLI turns into a WebdriverIO run. */
export interface RunnerConfig {
/** Directory holding the specs (default "tests"). */
Expand All @@ -150,6 +184,18 @@ export interface RunnerConfig {
};
/** Per-test timeout in ms (default 240000). */
mochaTimeout?: number;
/** Runner-native Mocha configuration. CLI `--grep` overrides only `mochaOpts.grep`. */
mochaOpts?: RunnerMochaOptions;
/** Number of retries for a failed spec file. */
specFileRetries?: number;
/** Delay in seconds before retrying a failed spec file. */
specFileRetriesDelay?: number;
/** Defer failed spec retries until the end of the queue. */
specFileRetriesDeferred?: boolean;
/** WebdriverIO reporters. Defaults to NativeProof's spec reporter. */
reporters?: RunnerReporter[];
/** Consumer lifecycle hook; failure capture still runs first. */
afterTest?: RunnerAfterTestHook;
/**
* WebdriverIO pass-throughs for tuning real-device runs. Each is forwarded only when set, so
* WebdriverIO's own defaults apply otherwise. Slow software-GPU emulators in particular often
Expand Down Expand Up @@ -185,6 +231,7 @@ export interface RunnerEnv {
platform?: string;
project?: string;
spec?: string;
grep?: string;
}

/** Pick the project by explicit name, else by platform (loudly failing on no match), else the first one. */
Expand Down Expand Up @@ -256,6 +303,21 @@ export function projectCapabilities(config: RunnerConfig, project: DeviceProject
};
}

type FailureCapture = (prefix: string) => Promise<unknown>;

/** @internal Compose public hooks without allowing them to replace default evidence capture. */
export function composeAfterTest(
consumerHook?: RunnerAfterTestHook,
captureFailure: FailureCapture = captureState,
): RunnerAfterTestHook {
return async (test, context, result) => {
if (!result.passed) {
await captureFailure(failureEvidenceName(test)).catch(() => {});
}
await consumerHook?.(test, context, result);
};
}

/**
* Translate an NativeProof config into a WebdriverIO `config` object.
*/
Expand All @@ -280,20 +342,18 @@ export function buildWdioConfig(
maxInstances: 1,
capabilities: [capabilities],
framework: "mocha",
reporters: ["spec"],
mochaOpts: { ui: "bdd", timeout: config.mochaTimeout ?? 240_000 },
reporters: config.reporters ?? ["spec"],
mochaOpts: {
ui: "bdd",
timeout: config.mochaTimeout ?? 240_000,
...config.mochaOpts,
...(env.grep !== undefined ? { grep: env.grep } : {}),
},
// Evidence on failure, out of the box: on a failed behaviour, snapshot a screenshot +
// redacted page source into the artifact dir, named after the spec. Best-effort — a
// capture error never masks the real failure — and consumers get it without writing
// their own afterTest hook.
afterTest: async (
test: { title: string; parent: string },
_context: unknown,
result: { passed: boolean },
): Promise<void> => {
if (result.passed) return;
await captureState(failureEvidenceName(test)).catch(() => {});
},
afterTest: composeAfterTest(config.afterTest),
};
// Optional WebdriverIO tuning — forwarded only when the consumer set it, so wdio's defaults apply
// otherwise (real emulators/simulators often need longer connection/wait timeouts than the defaults).
Expand All @@ -303,6 +363,10 @@ export function buildWdioConfig(
if (config.waitforTimeout !== undefined) wdio.waitforTimeout = config.waitforTimeout;
if (config.bail !== undefined) wdio.bail = config.bail;
if (config.logLevel !== undefined) wdio.logLevel = config.logLevel;
if (config.specFileRetries !== undefined) wdio.specFileRetries = config.specFileRetries;
if (config.specFileRetriesDelay !== undefined) wdio.specFileRetriesDelay = config.specFileRetriesDelay;
if (config.specFileRetriesDeferred !== undefined)
wdio.specFileRetriesDeferred = config.specFileRetriesDeferred;
return wdio;
}

Expand Down
12 changes: 11 additions & 1 deletion src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,17 @@ export type {
SessionContext,
} from "./app.js";
export { defineApp } from "./app.js";
export type { AppiumOptions, DeviceProject, NativeProofConfig, RunnerConfig } from "./config.js";
export type {
AppiumOptions,
DeviceProject,
NativeProofConfig,
RunnerAfterTestHook,
RunnerConfig,
RunnerMochaOptions,
RunnerReporter,
RunnerTest,
RunnerTestResult,
} from "./config.js";
export { defineConfig } from "./config.js";
export type { Driver, Platform } from "./driver.js";
export { wdioDriver } from "./driver.js";
Expand Down
2 changes: 2 additions & 0 deletions src/runner-env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -42,5 +42,7 @@ export function runnerEnvFromProcess(
const spec = envValue(env, "NATIVEPROOF_SPEC", "SPEC", warn);
if (spec) selection.spec = spec;
}
const grep = env.NATIVEPROOF_GREP;
if (grep !== undefined) selection.grep = grep;
return selection;
}
19 changes: 17 additions & 2 deletions test/cli.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ test("parseArgs defaults to the test command with sensible defaults", () => {
assert.equal(args.initPlatform, undefined);
});

test("parseArgs reads platform, project, spec and --no-appium", () => {
test("parseArgs reads platform, project, spec, grep and --no-appium", () => {
const args = parseArgs([
"test",
"--platform",
Expand All @@ -61,12 +61,15 @@ test("parseArgs reads platform, project, spec and --no-appium", () => {
"tablet",
"--spec",
"a.spec.ts",
"--grep",
"@smoke",
"--no-appium",
]);
assert.equal(args.platform, "ios");
assert.equal(args.initPlatform, "ios");
assert.equal(args.project, "tablet");
assert.equal(args.spec, "a.spec.ts");
assert.equal(args.grep, "@smoke");
assert.equal(args.startAppium, false);
});

Expand Down Expand Up @@ -246,15 +249,27 @@ test("loadNativeProofConfig errors when the config has no default export", async

test("helpText names the native E2E layer and version is semver", () => {
assert.match(helpText(), /Playwright-feeling native mobile E2E/);
assert.match(helpText(), /--grep <pattern>/);
assert.match(version(), /^\d+\.\d+\.\d+$/);
});

test("package carries the runtime reporter dependency used by generated WDIO config", () => {
test("runnerEnv forwards grep through a NativeProof-prefixed variable", () => {
const env = runnerEnv(parseArgs(["--grep", "@smoke"]), {});
assert.equal(env.NATIVEPROOF_GREP, "@smoke");
});

test("runnerEnv lets an explicit empty --grep clear an inherited filter", () => {
const env = runnerEnv(parseArgs(["--grep", ""]), { NATIVEPROOF_GREP: "@inherited" });
assert.equal(env.NATIVEPROOF_GREP, "");
});

test("package carries the runtime reporter and public WDIO type dependencies", () => {
const pkg = JSON.parse(readFileSync(path.join(process.cwd(), "package.json"), "utf8")) as {
dependencies?: Record<string, string>;
};

assert.equal(typeof pkg.dependencies?.["@wdio/spec-reporter"], "string");
assert.equal(typeof pkg.dependencies?.["@wdio/types"], "string");
});

test("Appium driver helpers map platforms and parse installed-driver output", () => {
Expand Down
Loading