From 93dc59886922b277a378d835e504e9c04e1ef5e6 Mon Sep 17 00:00:00 2001 From: John Doe <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 17:44:06 +0100 Subject: [PATCH] fork CI: each listed Python test declares its format, judged by it alone (#702) - .github/fork-ci-tests.txt: a Python test's line ends in "unittest" or "ok lines". A Python line with neither is refused with what to write; either word on a non-Python test is refused. All 87 listed Python tests now say theirs (78 unittest, 9 ok lines). - countTests: a Python test with no declared format counts nothing; it is never read as either. - fork-contracts.test.py: its record fixtures pass the format the list would give. Co-Authored-By: Claude Opus 5.5 --- .github/fork-ci-tests.txt | 163 ++++++++++---------- .github/scripts/fork-ci-tests.cjs | 41 +++-- .github/scripts/fork-ci-tests.test.cjs | 200 +++++++++++++++++++------ .github/scripts/fork-contracts.test.py | 9 +- 4 files changed, 269 insertions(+), 144 deletions(-) diff --git a/.github/fork-ci-tests.txt b/.github/fork-ci-tests.txt index 3e5bb95306836..9c73396168eb7 100644 --- a/.github/fork-ci-tests.txt +++ b/.github/fork-ci-tests.txt @@ -11,8 +11,11 @@ # runner's speed), fails its job, which says the weight to write. # run by run by , which must name the test's path (not one # of upstream's workflows: they are disabled in the fork) -# ... ok lines (after any of those, a Python test only) it prints ok lines, below; -# a Python test without it is a unittest test +# ... unittest (after any of those, a Python test only, which must say one of these +# ... ok lines two) its format: its tests are unittest tests, counted through +# fork-ci-unittest.py, or it prints ok lines, below. The runner judges +# it by this word alone: a unittest test that wrote no plan line, or an +# ok-lines test without its end line, counts nothing # A test's own `fork-ci:` comment lines say more (the runner reads them): # fork-ci: run [as root] [with _TESTS=value ...] [only [.]] # an own run, by `fork-ci-tests.cjs --run ` in fork-tests.yml; @@ -36,19 +39,19 @@ # A present test that is not listed fails. A listed file that is not present yet # is a note; one that was present and then deleted fails. .github/scripts/fork-ci-tests.test.cjs ~11s -.github/scripts/fork-checks-ran.test.py -.github/scripts/fork-ci-pressure.test.py -.github/scripts/fork-ci-rerun.test.py +.github/scripts/fork-checks-ran.test.py unittest +.github/scripts/fork-ci-pressure.test.py unittest +.github/scripts/fork-ci-rerun.test.py unittest .github/scripts/fork-contracts.test.py ~45s ok lines -.github/scripts/fork-fleet-desk-agent.test.py ~23s -.github/scripts/fork-fleet-desk-key.test.py ~10s -.github/scripts/fork-fleet-desk-seed.test.py ~34s -.github/scripts/fork-fleet-desk.test.py ~9s -.github/scripts/fork-fleet.test.py ~136s -.github/scripts/fork-fleet-refresh.test.py ~59s -.github/scripts/fork-fleet-walls.test.py ~71s -.github/scripts/fork-fleet-walls-contract.test.py ~96s -.github/scripts/fork-fleet-walls-wsl2.test.py ~40s +.github/scripts/fork-fleet-desk-agent.test.py ~23s unittest +.github/scripts/fork-fleet-desk-key.test.py ~10s unittest +.github/scripts/fork-fleet-desk-seed.test.py ~34s unittest +.github/scripts/fork-fleet-desk.test.py ~9s unittest +.github/scripts/fork-fleet.test.py ~136s unittest +.github/scripts/fork-fleet-refresh.test.py ~59s unittest +.github/scripts/fork-fleet-walls.test.py ~71s unittest +.github/scripts/fork-fleet-walls-contract.test.py ~96s unittest +.github/scripts/fork-fleet-walls-wsl2.test.py ~40s unittest .github/scripts/fork-gate-trust-docs.test.py ok lines .github/scripts/fork-head-gate-workflow.test.py ok lines .github/scripts/fork-follow.test.sh ~41s @@ -71,79 +74,79 @@ .github/scripts/fork-upstream-files-check.test.cjs .github/scripts/fork-vitest-gc-check.test.cjs .github/scripts/fork-workflow-triggers.test.cjs -fork-ci/actions/ci-demand/ci_demand.test.py ~17s -fork-ci/actions/docs-scope/docs-scope.test.py ~12s -fork-ci/actions/fleet-route/fleet_route.test.py -fork-ci/actions/fleet-watch/fleet_watch.test.py -fork-ci/actions/git-clean/git_clean.test.py ~8s -fork-ci/actions/gradle-cache/gradle_cache.test.py -fork-ci/actions/handoff-put/handoff.test.py ~16s -fork-ci/actions/head-check/head_check.test.py ~69s -fork-ci/actions/head-check/head_gate.test.py -fork-ci/actions/job-changed/job-changed.test.py ~21s -fork-ci/actions/pinned/pinned.test.py -fork-ci/actions/pr-queue/lens.test.py -fork-ci/actions/pr-queue/pr_queue.test.py ~42s -fork-ci/actions/pr-queue/pr_queue_laws.test.py ~23s -fork-ci/actions/pr-queue/review_config.test.py -fork-ci/actions/pr-queue/review_glob.test.py -fork-ci/actions/pr-queue/review_judge.test.py -fork-ci/actions/pr-queue/tested-base/tested_base.test.py -fork-ci/actions/test-select/test-select.test.py ~26s -fork-ci/actions/test-select/trace-check.test.py -fork-ci/actions/weakened-tests/weakened_tests.test.py -fork-ci/laws/laws.test.py -fork-kit/cleaner/tests/compress.test.py ~15s -fork-kit/cleaner/tests/daily.test.py ~19s -fork-kit/cleaner/tests/dedupe.test.py ~22s -fork-kit/cleaner/tests/engine.test.py -fork-kit/cleaner/tests/guard.test.py ~12s -fork-kit/cleaner/tests/install.test.py ~11s -fork-kit/cleaner/tests/report.test.py -fork-kit/cleaner/tests/scratch.test.py -fork-kit/cleaner/tests/worktrees.test.py ~43s -fork-kit/common/tests/free_contract.test.py ~6s -fork-kit/common/tests/kit_install.test.py ~93s -fork-kit/common/tests/machine_file.test.py -fork-kit/desk/tests/install.test.py ~103s -fork-kit/desk/tests/windows.test.py ~65s -fork-kit/golive/tests/back.test.py ~100s -fork-kit/golive/tests/check.test.py -fork-kit/golive/tests/contract.test.py -fork-kit/golive/tests/copy.test.py ~36s -fork-kit/golive/tests/gap.test.py ~84s -fork-kit/golive/tests/install.test.py ~168s -fork-kit/golive/tests/journal.test.py -fork-kit/golive/tests/rehearse.test.py ~56s -fork-kit/golive/tests/switch.test.py ~109s -fork-kit/gh-apps/tests/install.test.py -fork-kit/gh-apps/tests/mint.test.py -fork-kit/gh-apps/tests/policy.test.py -fork-kit/gh-apps/tests/run.test.py -fork-kit/gh-apps/tests/wrapper.test.py +fork-ci/actions/ci-demand/ci_demand.test.py ~17s unittest +fork-ci/actions/docs-scope/docs-scope.test.py ~12s unittest +fork-ci/actions/fleet-route/fleet_route.test.py unittest +fork-ci/actions/fleet-watch/fleet_watch.test.py unittest +fork-ci/actions/git-clean/git_clean.test.py ~8s unittest +fork-ci/actions/gradle-cache/gradle_cache.test.py unittest +fork-ci/actions/handoff-put/handoff.test.py ~16s unittest +fork-ci/actions/head-check/head_check.test.py ~69s unittest +fork-ci/actions/head-check/head_gate.test.py unittest +fork-ci/actions/job-changed/job-changed.test.py ~21s unittest +fork-ci/actions/pinned/pinned.test.py unittest +fork-ci/actions/pr-queue/lens.test.py unittest +fork-ci/actions/pr-queue/pr_queue.test.py ~42s unittest +fork-ci/actions/pr-queue/pr_queue_laws.test.py ~23s unittest +fork-ci/actions/pr-queue/review_config.test.py unittest +fork-ci/actions/pr-queue/review_glob.test.py unittest +fork-ci/actions/pr-queue/review_judge.test.py unittest +fork-ci/actions/pr-queue/tested-base/tested_base.test.py unittest +fork-ci/actions/test-select/test-select.test.py ~26s unittest +fork-ci/actions/test-select/trace-check.test.py unittest +fork-ci/actions/weakened-tests/weakened_tests.test.py unittest +fork-ci/laws/laws.test.py unittest +fork-kit/cleaner/tests/compress.test.py ~15s unittest +fork-kit/cleaner/tests/daily.test.py ~19s unittest +fork-kit/cleaner/tests/dedupe.test.py ~22s unittest +fork-kit/cleaner/tests/engine.test.py unittest +fork-kit/cleaner/tests/guard.test.py ~12s unittest +fork-kit/cleaner/tests/install.test.py ~11s unittest +fork-kit/cleaner/tests/report.test.py unittest +fork-kit/cleaner/tests/scratch.test.py unittest +fork-kit/cleaner/tests/worktrees.test.py ~43s unittest +fork-kit/common/tests/free_contract.test.py ~6s unittest +fork-kit/common/tests/kit_install.test.py ~93s unittest +fork-kit/common/tests/machine_file.test.py unittest +fork-kit/desk/tests/install.test.py ~103s unittest +fork-kit/desk/tests/windows.test.py ~65s unittest +fork-kit/golive/tests/back.test.py ~100s unittest +fork-kit/golive/tests/check.test.py unittest +fork-kit/golive/tests/contract.test.py unittest +fork-kit/golive/tests/copy.test.py ~36s unittest +fork-kit/golive/tests/gap.test.py ~84s unittest +fork-kit/golive/tests/install.test.py ~168s unittest +fork-kit/golive/tests/journal.test.py unittest +fork-kit/golive/tests/rehearse.test.py ~56s unittest +fork-kit/golive/tests/switch.test.py ~109s unittest +fork-kit/gh-apps/tests/install.test.py unittest +fork-kit/gh-apps/tests/mint.test.py unittest +fork-kit/gh-apps/tests/policy.test.py unittest +fork-kit/gh-apps/tests/run.test.py unittest +fork-kit/gh-apps/tests/wrapper.test.py unittest # Real users and sudo on the throw-away runner, as root (it refuses anywhere else). -fork-kit/gh-apps/tests/root.test.py run by .github/workflows/fork-tests.yml -fork-kit/heavy/tests/install.test.py ~8s -fork-kit/heavy/tests/measure.test.py ~10s -fork-kit/heavy/tests/queue.test.py ~18s -fork-kit/heavy/tests/sweep.test.py ~37s +fork-kit/gh-apps/tests/root.test.py run by .github/workflows/fork-tests.yml unittest +fork-kit/heavy/tests/install.test.py ~8s unittest +fork-kit/heavy/tests/measure.test.py ~10s unittest +fork-kit/heavy/tests/queue.test.py ~18s unittest +fork-kit/heavy/tests/sweep.test.py ~37s unittest # A real systemd user manager (the step starts one for the runner user). -fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml -fork-kit/model-guard/tests/guard.test.py -fork-kit/model-guard/tests/install.test.py ~21s -fork-kit/pc-agents/tests/check.test.py ~32s -fork-kit/pc-agents/tests/setup.test.py ~153s -fork-kit/pc-agents/tests/update.test.py ~66s -fork-kit/schedule-kick/tests/cron.test.py -fork-kit/schedule-kick/tests/install.test.py -fork-kit/schedule-kick/tests/kick.test.py -fork-kit/schedule-kick/tests/late.test.py +fork-kit/heavy/tests/heavy.test.py run by .github/workflows/fork-tests.yml unittest +fork-kit/model-guard/tests/guard.test.py unittest +fork-kit/model-guard/tests/install.test.py ~21s unittest +fork-kit/pc-agents/tests/check.test.py ~32s unittest +fork-kit/pc-agents/tests/setup.test.py ~153s unittest +fork-kit/pc-agents/tests/update.test.py ~66s unittest +fork-kit/schedule-kick/tests/cron.test.py unittest +fork-kit/schedule-kick/tests/install.test.py unittest +fork-kit/schedule-kick/tests/kick.test.py unittest +fork-kit/schedule-kick/tests/late.test.py unittest # Upstream's tests, run where fork-tests.yml follows upstream's ci.yml. .github/scripts/check-nightly-release.test.cjs run by .github/workflows/fork-tests.yml .github/scripts/mobile-preview-comment.test.cjs run by .github/workflows/fork-tests.yml .github/scripts/mobile-ios-preview-record.test.cjs run by .github/workflows/fork-tests.yml -.github/scripts/stage-preview-bundle.test.py run by .github/workflows/fork-tests.yml +.github/scripts/stage-preview-bundle.test.py run by .github/workflows/fork-tests.yml unittest .github/scripts/test-delta.test.cjs run by .github/workflows/fork-tests.yml .github/scripts/vercel-deployment-output.test.cjs run by .github/workflows/fork-tests.yml # Upstream runs this one from thread-transfer-report.yml (workflow_run of the disabled CI). diff --git a/.github/scripts/fork-ci-tests.cjs b/.github/scripts/fork-ci-tests.cjs index 86d4302a2c676..e96c18608cbee 100644 --- a/.github/scripts/fork-ci-tests.cjs +++ b/.github/scripts/fork-ci-tests.cjs @@ -171,24 +171,32 @@ function readList(text) { .map((line) => line.replace(/#.*/, "").trim()) .filter(Boolean) .map((line) => { - const match = line.match(/^(\S+)(?:\s+~([1-9]\d{0,4})s|\s+run by\s+(\S+))?(\s+ok lines)?$/); + const match = line.match( + /^(\S+)(?:\s+~([1-9]\d{0,4})s|\s+run by\s+(\S+))?(?:\s+(unittest|ok lines))?$/, + ); if (!match) throw new Error( `${LIST}: cannot read "${line}" (expected "", " ~s" or " run by ", ` + - `each optionally ending in "ok lines")`, + `a Python test's then ending in its format: "unittest" or "ok lines")`, + ); + // A Python test has two formats (countTests()), and says which: none is read as the other. + const python = match[1].endsWith(".py"); + if (match[4] && !python) + throw new Error( + `${LIST}: "${line}": only a Python test says "${match[4]}" (a shell test always prints ok lines, a ` + + `node test never does)`, ); - // Only a Python test has two formats (countTests()): unittest unless it says "ok lines". - if (match[4] && !match[1].endsWith(".py")) + if (!match[4] && python) throw new Error( - `${LIST}: "${line}": only a Python test says "ok lines" (a shell test always prints them, a node test ` + - `never does)`, + `${LIST}: "${line}": a Python test says its format at the end of its line: "unittest" (its tests ` + + `are unittest tests) or "ok lines" (it prints ok lines and its end line)`, ); return { file: match[1], // `./a/b` and `a//b` name the same file as `a/b`, so every reader compares one spelling. runBy: match[3] ? path.posix.normalize(match[3]) : null, weight: match[2] ? Number(match[2]) : null, - format: match[1].endsWith(".py") ? (match[4] ? "ok lines" : "unittest") : null, + format: match[4] ?? null, }; }); } @@ -446,9 +454,9 @@ const isEnd = (u) => u.unloaded.every((n) => typeof n === "string" && TEST_NAME.test(n)); // What a test ran, { format, tests, skipped, names, unloaded, failed, note }, or null (no count; tests null: none known -// to have run, `note` says why). `how`: the list's Python format and the own run's `only`. Python unittest: the shim's -// plan and counts per run, then its end line, whose `unloaded` tests count as skipped by name; node: its one line; -// else stdout's ok, FAIL and `skip ` lines, ending in OK_LINES_FORM with the same counts. +// to have run, `note` says why). `how`: the list's Python format (none: no count) and the own run's `only`. Python +// unittest: the shim's plan and counts per run, then its end line, whose `unloaded` tests count as skipped by name; +// node: its one line; else stdout's ok, FAIL and `skip ` lines, ending in OK_LINES_FORM with the same counts. function countTests(file, stdout, stderr, lines = [], how = {}) { const ext = file.match(TEST_FILE)?.[1]; const whole = (n) => Number.isInteger(n) && n >= 0; @@ -461,7 +469,12 @@ function countTests(file, stdout, stderr, lines = [], how = {}) { failed: null, note, }); - const format = ext === "py" ? (how.format ?? "unittest") : null; + const format = ext === "py" ? (how.format ?? null) : null; + if (ext === "py" && format !== "unittest" && format !== "ok lines") + return none( + null, + `${LIST} declares no format for it ("unittest" or "ok lines"), so it cannot be read`, + ); const shimEnd = ext === "py" && isEnd(lines.at(-1)) ? lines.at(-1) : null; if (format === "ok lines") { if (lines.length !== 1 || !shimEnd) @@ -483,7 +496,7 @@ function countTests(file, stdout, stderr, lines = [], how = {}) { return none( "unittest", "fork-ci-unittest.py wrote no line: it stopped hard before its first unittest run (while it loads?) and " + - `never reached its end; a Python test that prints ok lines says "ok lines" in ${LIST}`, + `never reached its end; a Python test that prints ok lines says "ok lines" in ${LIST}, not "unittest"`, ); const runs = []; const open = []; @@ -515,7 +528,7 @@ function countTests(file, stdout, stderr, lines = [], how = {}) { return none( "unittest", "it started no unittest run (an exit before unittest.main?); a Python test that prints ok lines says " + - `"ok lines" in ${LIST}`, + `"ok lines" in ${LIST}, not "unittest"`, ); // An own run's `only`: one class (its tests and its nested classes' count), or one test (that test only). const unloaded = how.only @@ -636,7 +649,7 @@ function recordProblems(record) { } // `allowed`: the tests the run may and must skip, from the test's declarations (the plan), never from what ran. -// `format`: a Python test's, as the list declares it (readList(); none: unittest). +// `format`: a Python test's, as the list declares it (readList(); none: no count). function record(key, file, run, result, allowed, format = null) { const counts = countTests(file, result.stdout ?? "", result.stderr ?? "", result.unittest ?? [], { format, diff --git a/.github/scripts/fork-ci-tests.test.cjs b/.github/scripts/fork-ci-tests.test.cjs index 3c5361e37a66a..440da9ad940eb 100644 --- a/.github/scripts/fork-ci-tests.test.cjs +++ b/.github/scripts/fork-ci-tests.test.cjs @@ -205,31 +205,51 @@ test("the real release-smoke.ts names its listed test outside a comment", () => test("the list ignores comments and blank lines and reads `run by` and weights", () => { assert.deepEqual( - readList("# head\n\na.test.sh # from #2\n b.test.py run by c.yml\nd.test.cjs ~120s\n"), + readList( + "# head\n\na.test.sh # from #2\n b.test.py run by c.yml unittest\nd.test.cjs ~120s\n", + ), [ { file: "a.test.sh", runBy: null, weight: null, format: null }, { file: "b.test.py", runBy: "c.yml", weight: null, format: "unittest" }, { file: "d.test.cjs", runBy: null, weight: 120, format: null }, ], ); - // A Python test is a unittest test unless it says it prints ok lines, after any other word. + // A Python test says its format, "unittest" or "ok lines", after any other word. assert.deepEqual( readList( - "e.test.py ok lines\nf.test.py ~40s ok lines\ng.test.py run by c.yml ok lines\nh.test.py ~40s\n", + "e.test.py ok lines\nf.test.py ~40s ok lines\ng.test.py run by c.yml ok lines\nh.test.py ~40s unittest\n" + + "i.test.py unittest\n", ).map((e) => [e.file, e.weight, e.runBy, e.format]), [ ["e.test.py", null, null, "ok lines"], ["f.test.py", 40, null, "ok lines"], ["g.test.py", null, "c.yml", "ok lines"], ["h.test.py", 40, null, "unittest"], + ["i.test.py", null, null, "unittest"], ], ); - for (const other of ["a.test.sh ok lines", "a.test.cjs ~20s ok lines"]) - assert.throws(() => readList(`${other}\n`), /only a Python test says "ok lines"/, other); + for (const [other, word] of [ + ["a.test.sh ok lines", "ok lines"], + ["a.test.cjs ~20s ok lines", "ok lines"], + ["a.test.sh unittest", "unittest"], + ["a.test.mjs run by c.yml unittest", "unittest"], + ]) + assert.throws( + () => readList(`${other}\n`), + new RegExp(`only a Python test says "${word}"`), + other, + ); + // A Python test without its format is refused, whatever else its line says: none is read as unittest. + for (const bare of ["a.test.py", "a.test.py ~40s", "a.test.py run by c.yml"]) + assert.throws( + () => readList(`b.test.py unittest\n${bare}\n`), + /a Python test says its format at the end of its line: "unittest" .* or "ok lines"/, + bare, + ); // One spelling of a runner's path, so `./a/b` and `a//b` compare equal to `a/b` wherever the list is read. assert.deepEqual( readList( - "e.test.py run by ./.github/workflows/x.yml\nf.test.py run by .github//workflows/x.yml\n", + "e.test.py run by ./.github/workflows/x.yml unittest\nf.test.py run by .github//workflows/x.yml ok lines\n", ).map((e) => e.runBy), [".github/workflows/x.yml", ".github/workflows/x.yml"], ); @@ -242,6 +262,11 @@ test("the list ignores comments and blank lines and reads `run by` and weights", "a.test.py ok lines ~12s", "a.test.py oklines", "a.test.py ok lines ok lines", + "a.test.py unittest ~12s", + "a.test.py unittest ok lines", + "a.test.py ok lines unittest", + "a.test.py unittests", + "a.test.py Unittest", ]) assert.throws(() => readList(`${bad}\n`), /cannot read/, bad); }); @@ -860,11 +885,13 @@ test("no listed shell test pipes into a reader that stops early", () => { test("countTests reads fork-ci-unittest.py's plan and count lines, node's reporter line, else ok lines", () => { const py = "fork-ci/a.test.py"; + // The list's format for a Python test (readList() refuses one without). + const ut = { format: "unittest" }; const line = (tests, skipped = [], failed = 0) => ({ tests, skipped, failed }); const plan = (n) => ({ plan: n }); // The shim's last line, when the file ended: the tests its source defines that no run planned. const end = (...unloaded) => ({ unloaded }); - assert.deepEqual(countTests(py, "", "", [plan(3), line(3, ["T.test_b"]), end()]), { + assert.deepEqual(countTests(py, "", "", [plan(3), line(3, ["T.test_b"]), end()], ut), { format: "unittest", tests: 3, skipped: 1, @@ -878,7 +905,7 @@ test("countTests reads fork-ci-unittest.py's plan and count lines, node's report [plan(2), line(2, [], 1), plan(4), line(4, ["U.test_x", "U.test_y"], 2), end()], [plan(4), plan(2), line(2, [], 1), line(4, ["U.test_x", "U.test_y"], 2), end()], ]) - assert.deepEqual(countTests(py, "", "", lines), { + assert.deepEqual(countTests(py, "", "", lines, ut), { format: "unittest", tests: 6, skipped: 2, @@ -889,33 +916,42 @@ test("countTests reads fork-ci-unittest.py's plan and count lines, node's report }); // A test its source defines that no run planned (a class only under a condition, a load_tests that drops it) is a // skip by its name; an own run of one class counts only that class's. - assert.deepEqual(countTests(py, "", "", [plan(2), line(2), end("Tool.test_a", "Tool.test_b")]), { - format: "unittest", - tests: 4, - skipped: 2, - names: ["Tool.test_a", "Tool.test_b"], - unloaded: ["Tool.test_a", "Tool.test_b"], - failed: 0, - note: null, - }); + assert.deepEqual( + countTests(py, "", "", [plan(2), line(2), end("Tool.test_a", "Tool.test_b")], ut), + { + format: "unittest", + tests: 4, + skipped: 2, + names: ["Tool.test_a", "Tool.test_b"], + unloaded: ["Tool.test_a", "Tool.test_b"], + failed: 0, + note: null, + }, + ); const only = countTests(py, "", "", [plan(1), line(1), end("Tool.test_a", "L.test_b")], { + ...ut, only: "L", }); assert.deepEqual([only.tests, only.names, only.unloaded], [2, ["L.test_b"], ["L.test_b"]]); // A hard stop before the first plan (at import, while it loads): no line at all, whatever its stdout says (a // child's ok lines and end line). Not an ok-lines test unless the list says so. - for (const how of [{}, { format: "unittest" }]) { - const counted = countTests(py, "ok child\nall passed: 1 ok, 0 skipped\n", "", [], how); - assert.deepEqual([counted.format, counted.tests], ["unittest", null]); - assert.match(counted.note, /wrote no line: it stopped hard before its first unittest run/); - } + const hard = countTests(py, "ok child\nall passed: 1 ok, 0 skipped\n", "", [], ut); + assert.deepEqual([hard.format, hard.tests], ["unittest", null]); + assert.match(hard.note, /wrote no line: it stopped hard before its first unittest run/); + // A Python test with no format from the list is never read as either: nothing counts, whatever its lines. + for (const lines of [[], [plan(1), line(1), end()], [end()]]) + for (const how of [{}, { format: null }, { format: "ok" }, { only: "T" }]) { + const counted = countTests(py, "ok child\nall passed: 1 ok, 0 skipped\n", "", lines, how); + assert.deepEqual([counted.format, counted.tests], [null, null], JSON.stringify([lines, how])); + assert.match(counted.note, /declares no format for it \("unittest" or "ok lines"\)/); + } // Its runs ended, then a hard stop before the file's end; or the file ended without a run (sys.exit at import). for (const [lines, note] of [ [[plan(2), line(2)], /runs ended, but the file never reached its end/], [[end()], /started no unittest run/], [[end("T.test_a")], /started no unittest run/], ]) { - const counted = countTests(py, "ok child\nall passed: 1 ok, 0 skipped\n", "", lines); + const counted = countTests(py, "ok child\nall passed: 1 ok, 0 skipped\n", "", lines, ut); assert.deepEqual([counted.format, counted.tests], ["unittest", null], JSON.stringify(lines)); assert.match(counted.note, note); } @@ -926,23 +962,23 @@ test("countTests reads fork-ci-unittest.py's plan and count lines, node's report [plan(4), plan(2), line(2)], [plan(4), end()], ]) { - const counted = countTests(py, "ok child\nall passed: 1 ok, 0 skipped\n", "", lines); + const counted = countTests(py, "ok child\nall passed: 1 ok, 0 skipped\n", "", lines, ut); assert.equal(counted.format, "unittest"); assert.equal(counted.tests, null); assert.match(counted.note, /started but never ended/); } // Stopped half-way through writing a line, counts without their plan, a count not its plan's, or another shape. - assert.equal(countTests(py, "", "", [plan(2), line(2), null]), null); - assert.equal(countTests(py, "", "", [line(2)]), null); - assert.equal(countTests(py, "", "", [plan(3), line(2)]), null); - assert.equal(countTests(py, "", "", [plan(2), { tests: 2, skipped: 1, failed: 0 }]), null); - assert.equal(countTests(py, "", "", [{ plan: 2, tests: 2 }, line(2)]), null); - assert.equal(countTests(py, "", "", [plan(2), line(2), end(), end()]), null); - assert.equal(countTests(py, "", "", [end(), plan(2), line(2)]), null); - assert.equal(countTests(py, "", "", [plan(2), line(2), { unloaded: [1] }]), null); - assert.equal(countTests(py, "", "", [plan(2), line(2), { unloaded: [], more: 1 }]), null); + assert.equal(countTests(py, "", "", [plan(2), line(2), null], ut), null); + assert.equal(countTests(py, "", "", [line(2)], ut), null); + assert.equal(countTests(py, "", "", [plan(3), line(2)], ut), null); + assert.equal(countTests(py, "", "", [plan(2), { tests: 2, skipped: 1, failed: 0 }], ut), null); + assert.equal(countTests(py, "", "", [{ plan: 2, tests: 2 }, line(2)], ut), null); + assert.equal(countTests(py, "", "", [plan(2), line(2), end(), end()], ut), null); + assert.equal(countTests(py, "", "", [end(), plan(2), line(2)], ut), null); + assert.equal(countTests(py, "", "", [plan(2), line(2), { unloaded: [1] }], ut), null); + assert.equal(countTests(py, "", "", [plan(2), line(2), { unloaded: [], more: 1 }], ut), null); // unittest's own summary is never read (a class skip counts once there; stderr may go on after it). - assert.equal(countTests(py, "", "Ran 3 tests in 0.01s\n\nOK (skipped=1)\n", []).tests, null); + assert.equal(countTests(py, "", "Ran 3 tests in 0.01s\n\nOK (skipped=1)\n", [], ut).tests, null); const js = "fork-ci/a.test.cjs"; const node = (tests, more = {}) => ({ tests, @@ -1251,6 +1287,7 @@ test("record keeps what a run was, what its output counted and the skips its tes ], }, ["L.test_y", "L.test_x"], + "unittest", ), { key: "f --run only L", @@ -1271,7 +1308,7 @@ test("record keeps what a run was, what its output counted and the skips its tes allowed: ["L.test_x", "L.test_y"], }, ); - // A Python test's format is the list's (none given: unittest), so a hard stop as it loads is never read as ok lines. + // A Python test's format is the list's (none given: no count), so a hard stop as it loads is never read as ok lines. const printed = { status: 0, seconds: "1", @@ -1293,6 +1330,7 @@ test("record keeps what a run was, what its output counted and the skips its tes ], [null, null, 1], ); + assert.equal(record("p", "fork-ci/p.test.py", null, printed, []).format, null); const none = record("f", "fork-ci/f.test.sh", null, { status: 0, seconds: "0.1" }, ["b", "a"]); assert.deepEqual( [none.own, none.vars, none.tests, none.skipped, none.skipped_tests, none.allowed], @@ -1464,7 +1502,7 @@ test("a part writes one record of every test it ran, keyed by run, attempt, job " - run: node .github/scripts/fork-ci-tests.cjs --run fork-ci/own.test.py\n", }; const list = - "fork-ci/a.test.sh\nfork-ci/b.test.sh\nfork-ci/own.test.py run by .github/workflows/fork-tests.yml\n"; + "fork-ci/a.test.sh\nfork-ci/b.test.sh\nfork-ci/own.test.py run by .github/workflows/fork-tests.yml unittest\n"; const { run, records, cleanup } = runRecording(tests, ["fork-ci/"], list, { FORK_CI_TEST_SHARD: "2/2", }); @@ -1513,7 +1551,7 @@ test("a part fails on a skip its test does not declare or a declared one that ra const { run, records, cleanup } = runRecording( { "fork-ci/s.test.py": head + SKIPPING_PY }, [], - "fork-ci/s.test.py", + "fork-ci/s.test.py unittest", ); try { assert.equal(run.status, status, head + run.stdout + run.stderr); @@ -1634,7 +1672,7 @@ test("a class or module skip counts every test it skips, by name, so a skipped c const { run, records, cleanup } = runRecording( { "fork-ci/c.test.py": text }, [], - "fork-ci/c.test.py", + "fork-ci/c.test.py unittest", ); try { assert.equal(run.status, status, text + run.stdout + run.stderr); @@ -1660,7 +1698,7 @@ test("a Python test run through the shim imports its own folder's modules and se "if __name__ == '__main__':\n" + " unittest.main()\n", }; - const { run, records, cleanup } = runRecording(tests, [], "fork-ci/i.test.py"); + const { run, records, cleanup } = runRecording(tests, [], "fork-ci/i.test.py unittest"); try { assert.equal(run.status, 0, run.stdout + run.stderr); assert.equal(Object.values(readRecords(records))[0].results[0].tests, 3); @@ -1919,8 +1957,8 @@ test("a part's owed list is its plan (its shard's tests), not what came back fro }); // One test file run as a listed part test, recording: [exit status, its record, the part's stdout]. `line`: its line -// in the list. -function recordOf(file, text, more = {}, line = file) { +// in the list (a Python test's says "unittest" unless given). +function recordOf(file, text, more = {}, line = file.endsWith(".py") ? `${file} unittest` : file) { const { run, records, cleanup } = runRecording({ [file]: text, ...more }, [], line); try { return [run.status, Object.values(readRecords(records))[0].results[0], run.stdout + run.stderr]; @@ -2002,7 +2040,7 @@ test("a Python test that stops before its first unittest run counts nothing, wha ["os._exit(0)", /wrote no line: it stopped hard before its first unittest run/], ["sys.exit(0)", /started no unittest run/], ]) { - const [status, r, out] = recordOf(file, STOPS_AS_IT_LOADS(exit)); + const [status, r, out] = recordOf(file, STOPS_AS_IT_LOADS(exit), {}, `${file} unittest`); assert.equal(status, 1, exit + out); assert.deepEqual([r.format, r.tests], ["unittest", null], exit); assert.match(r.note, note, exit); @@ -2025,7 +2063,12 @@ test("a Python test that prints ok lines is read by them only where the list say ); assert.equal(status, 0, out); assert.deepEqual([r.format, r.tests, r.skipped_tests, r.note], ["ok lines", 2, ["net"], null]); - const [status2, r2] = recordOf(file, "# fork-ci: may skip net\n" + OK_LINES_PY); + const [status2, r2] = recordOf( + file, + "# fork-ci: may skip net\n" + OK_LINES_PY, + {}, + `${file} unittest`, + ); assert.equal(status2, 1); assert.deepEqual([r2.format, r2.tests], ["unittest", null]); assert.match( @@ -2040,7 +2083,7 @@ test("a Python test that prints ok lines is read by them only where the list say const plain = "print('ok one')\nprint('all passed: 1 ok, 0 skipped')\n"; for (const [line, status4, format] of [ [`${file} run by .github/workflows/fork-tests.yml ok lines`, 0, "ok lines"], - [`${file} run by .github/workflows/fork-tests.yml`, 1, "unittest"], + [`${file} run by .github/workflows/fork-tests.yml unittest`, 1, "unittest"], ]) { const { run, records, cleanup } = runRecording({ [file]: plain }, ["--run", file], line); try { @@ -2053,6 +2096,71 @@ test("a Python test that prints ok lines is read by them only where the list say } }); +// A unittest file that stops hard as it loads, before any unittest run: no shim line, nothing on stdout. +const EXITS_AT_IMPORT = + "import os\n" + + "os._exit(0)\n" + + "import unittest\n" + + "class T(unittest.TestCase):\n" + + " def test_a(self): pass\n" + + "unittest.main()\n"; + +test("each listed Python test declares its format and is judged by it alone; none declared is refused (#702)", () => { + const file = "fork-ci/d.test.py"; + // What a record says, without its time: a run again must give the same. + const verdict = ([status, r]) => [status, r.format, r.tests, r.skipped_tests, r.note]; + // Declared unittest, it stops as it loads: no plan line, red; the same again on a second run. + const first = verdict(recordOf(file, EXITS_AT_IMPORT, {}, `${file} unittest`)); + assert.deepEqual(first.slice(0, 4), [1, "unittest", null, []]); + assert.match(first[4], /wrote no line: it stopped hard before its first unittest run/); + assert.deepEqual(verdict(recordOf(file, EXITS_AT_IMPORT, {}, `${file} unittest`)), first); + // Declared ok lines, an ok-lines file passes, by its own count, again on a second run. + const ok = verdict( + recordOf(file, "# fork-ci: may skip net\n" + OK_LINES_PY, {}, `${file} ok lines`), + ); + assert.deepEqual(ok, [0, "ok lines", 2, ["net"], null]); + assert.deepEqual( + verdict(recordOf(file, "# fork-ci: may skip net\n" + OK_LINES_PY, {}, `${file} ok lines`)), + ok, + ); + // A declared format the file does not keep is red, either way round. + for (const [text, line, format, note] of [ + [OK_LINES_PY, `${file} unittest`, "unittest", /started no unittest run/], + [SKIPPING_PY, `${file} ok lines`, "ok lines", /says it prints ok lines, but it ran unittest/], + [EXITS_AT_IMPORT, `${file} ok lines`, "ok lines", /stopped before its end \(a hard exit/], + ]) { + const [status, r, out] = recordOf(file, text, {}, line); + assert.equal(status, 1, line + out); + assert.deepEqual([r.format, r.tests], [format, null], line); + assert.match(r.note, note, line); + } + // No format declared: the list is refused, with what to write, before any test runs (MARKS/ran stays absent). + for (const line of [file, `${file} ~20s`, `${file} run by .github/workflows/fork-tests.yml`]) { + const tests = { + [file]: + "import os\nopen(os.path.join(os.environ['MARKS'], 'ran'), 'w').close()\n" + OK_LINES_PY, + "fork-ci/a.test.sh": 'echo "ok a"\n' + END(1), + }; + const { root, run, records, cleanup } = runRecording( + tests, + line.includes("run by") ? ["--run", file] : [], + `fork-ci/a.test.sh\n${line}`, + ); + try { + assert.equal(run.status, 1, line + run.stdout + run.stderr); + assert.match( + run.stderr, + /fork-ci-tests\.txt: ".*d\.test\.py.*": a Python test says its format at the end of its line: "unittest"/, + line, + ); + assert.equal(fs.existsSync(path.join(root, "ran")), false, line); + assert.equal(fs.existsSync(path.join(records, "records")), false, line); + } finally { + cleanup(); + } + } +}); + test("a test the file defines that no unittest run loaded is a skip by name: a class under a condition, load_tests", () => { const file = "fork-ci/u.test.py"; const conditional = @@ -2123,7 +2231,7 @@ test("an own run of one class counts that class's unloaded tests only", () => { "class M(unittest.TestCase):\n" + " def test_m(self): pass\n" + "unittest.main()\n"; - const list = `${file} run by .github/workflows/fork-tests.yml`; + const list = `${file} run by .github/workflows/fork-tests.yml unittest`; for (const [cond, status, skipped] of [ ["True", 0, []], ["False", 1, ["L.test_b"]], @@ -2390,7 +2498,7 @@ test("--run refuses what it may not run, and an own run may skip no test", () => "fork-ci/part.test.sh", "fork-ci/other.test.sh run by .github/workflows/fork-other.yml", "fork-ci/bad.test.sh run by .github/workflows/fork-tests.yml", - "fork-ci/skip.test.py", + "fork-ci/skip.test.py unittest", "fork-ci/gone.test.sh", ].join("\n"); const refused = [ diff --git a/.github/scripts/fork-contracts.test.py b/.github/scripts/fork-contracts.test.py index 4d62684d95854..9274e2c54ff3e 100644 --- a/.github/scripts/fork-contracts.test.py +++ b/.github/scripts/fork-contracts.test.py @@ -4133,9 +4133,9 @@ def result_step(jobs, name): CHECK_SCRIPT = check_script(CHECK_STEP["run"]) REAL_RECORD_JOBS = json.loads(CHECK_STEP["env"]["RECORD_JOBS"]) FIX_RUN, FIX_SHA = "4242", "c0ffee" + "0" * 34 -# A Python test's result as the runner reads it: the lines fork-ci-unittest.py wrote (`unittest`: a plan, then its -# counts, each test by name, then its end line: the tests the file defines that no run loaded); a node test's, the -# line fork-ci-node-counts.cjs wrote. +# A Python test's result as the runner reads it, the list saying "unittest" for each (#702): the lines +# fork-ci-unittest.py wrote (`unittest`: a plan, then its counts, each test by name, then its end line: the tests the +# file defines that no run loaded); a node test's, the line fork-ci-node-counts.cjs wrote. def py(tests, skipped=(), unloaded=()): return {"status": 0, "seconds": "1.0", "stdout": "", "stderr": f"Ran {tests} tests in 0.1s\n\nOK\n", "unittest": [{"plan": tests}, {"tests": tests, "skipped": list(skipped), "failed": 0}, @@ -4203,7 +4203,8 @@ def good_records(): "for (const [name, calls] of Object.entries(artifacts)) { fs.mkdirSync(path.join(folder, name), " "{ recursive: true }); for (const c of calls) r.writeRecords({ dir: path.join(folder, name), run_id: c.run_id, " "run_attempt: c.attempt, job: c.job }, c.name, { sha: c.sha, call: c.call, all: c.all, owed: c.owed, " - "results: c.results.map(([key, file, run, result, allowed]) => r.record(key, file, run, result, allowed)) }); }") + "results: c.results.map(([key, file, run, result, allowed]) => r.record(key, file, run, result, allowed, " + "file.endsWith('.py') ? 'unittest' : null)) }); }") def run_check(artifacts, attempt=1, tweak=None, sha=FIX_SHA):