Corroborate Python no-test exit-code 5

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes are documented here. This project follows Semantic Versionin
 
 ### Fixed
 
+- Tightened Python test-runner exit-code 5 handling. Pytest and the stdlib `unittest` CLI use code 5 for no-test outcomes, but repository-local modules can also terminate runner imports with code 5 before trustworthy lifecycle observation. ProofDiff now accepts exit 5 as non-failing only when high-confidence exact zero-test observer data corroborates it: directly for targeted pytest, or through the paired targeted observer for opaque pytest/unittest commands. Uncorroborated exit-5 failures remain verification failures.
 - Preserved LCOV `SF:` source paths exactly instead of applying JavaScript-style whitespace trimming. Leading or trailing spaces and Unicode whitespace are valid path characters, so silently erasing them could alias artifact coverage for one path onto a different changed repository file and overstate changed-line evidence.
 - Restricted recognized Node/Jest/Vitest package-script trimming to ASCII spaces and tabs. JavaScript `String.trim()` also removes non-ASCII whitespace such as NBSP, which shells do not treat as ordinary script separators; leading or trailing Unicode whitespace can therefore no longer be erased into a different executable command and accidentally qualify for exact targeted evidence.
 - Stopped `.pyi` stub files with test-like names from inventing executable pytest or `unittest` checks. Stubs remain available to static Python analysis and relationship discovery, but framework discovery now requires an executable `.py` test file; this prevents `--run-checks` from launching a Python test runner merely because a repository contains test-shaped type stubs.

src/evidence.ts

@@ -11,8 +11,24 @@ function checkApplies(check: CheckResult, file: ChangedFile, relatedTests: strin
   return true;
 }
 
-function isRecognizedNoTestsExit(check: CheckResult): boolean {
-  return check.status === "failed" && check.exitCode === 5 && (check.targetRunner === "pytest" || check.targetRunner === "unittest");
+function hasExactZeroTestObservations(check: CheckResult): boolean {
+  const observations = check.targetObservations ?? [];
+  return observations.length > 0
+    && observations.every((observation) => observation.outcome === "zero-tests"
+      && check.targetQualifications?.some((qualification) => qualification.confidence === "high"
+        && qualification.path === observation.path
+        && qualification.runnerPath === observation.runnerPath) === true);
+}
+
+function isRecognizedNoTestsExit(check: CheckResult, checks: CheckResult[]): boolean {
+  if (check.status !== "failed" || check.exitCode !== 5 || (check.targetRunner !== "pytest" && check.targetRunner !== "unittest")) return false;
+  if (check.targetQualifications !== undefined) {
+    return check.targetRunner === "pytest" && hasExactZeroTestObservations(check);
+  }
+  const targeted = checks.find((candidate) => candidate.id === `${check.id}:targeted`);
+  if (targeted === undefined || !hasExactZeroTestObservations(targeted)) return false;
+  return targeted.status === "passed"
+    || (targeted.status === "failed" && targeted.exitCode === 5 && targeted.targetRunner === "pytest");
 }
 
 function verificationFor(file: ChangedFile, relatedTests: string[], checks: CheckResult[], declaredTests: string[]): { status: VerificationStatus; evidence: EvidenceItem[]; executedTests: string[]; testExecutions: FileAssessment["testExecutions"] } {
@@ -30,17 +46,17 @@ function verificationFor(file: ChangedFile, relatedTests: string[], checks: Chec
   const hasUnavailableRelatedTarget = (check: CheckResult): boolean => check.targetObservations?.some((observation) => relatedTests.includes(observation.path) && observation.outcome === "not-observed" && qualificationForObservation(check, observation)?.confidence === "high") === true;
   const localizedTargetedProcessFailures = applicable.filter((check) => check.targetQualifications !== undefined
     && check.status === "failed"
-    && !isRecognizedNoTestsExit(check)
+    && !isRecognizedNoTestsExit(check, applicable)
     && hasExactTargetFailure(check)
     && !hasUnavailableRelatedTarget(check));
   const unlocalizedTargetedFailures = applicable.filter((check) => check.targetQualifications !== undefined
     && check.status === "failed"
-    && !isRecognizedNoTestsExit(check)
+    && !isRecognizedNoTestsExit(check, applicable)
     && (!hasExactTargetFailure(check) || hasUnavailableRelatedTarget(check)));
   const opaqueFailures = applicable.filter((check) => check.targetQualifications === undefined
     && ["failed", "error", "timed-out"].includes(check.status)
     && !(check.kind === "test" && check.targetRunner !== undefined && localizedTargetedProcessFailures.some((targeted) => targeted.id === `${check.id}:targeted`))
-    && !isRecognizedNoTestsExit(check));
+    && !isRecognizedNoTestsExit(check, applicable));
   const operationalFailures = applicable.filter((check) => check.targetQualifications !== undefined && ["error", "timed-out"].includes(check.status));
   const passing = applicable.filter((check) => check.status === "passed" && check.targetQualifications === undefined);
   const testExecutions: FileAssessment["testExecutions"] = exactObservations

src/explanation.ts

@@ -7,8 +7,24 @@ function checkApplies(check: CheckResult, item: FileAssessment): boolean {
   return true;
 }
 
-function recognizedNoTestsExit(check: CheckResult): boolean {
-  return check.status === "failed" && check.exitCode === 5 && (check.targetRunner === "pytest" || check.targetRunner === "unittest");
+function hasExactZeroTestObservations(check: CheckResult): boolean {
+  const observations = check.targetObservations ?? [];
+  return observations.length > 0
+    && observations.every((observation) => observation.outcome === "zero-tests"
+      && check.targetQualifications?.some((qualification) => qualification.confidence === "high"
+        && qualification.path === observation.path
+        && qualification.runnerPath === observation.runnerPath) === true);
+}
+
+function recognizedNoTestsExit(check: CheckResult, checks: CheckResult[]): boolean {
+  if (check.status !== "failed" || check.exitCode !== 5 || (check.targetRunner !== "pytest" && check.targetRunner !== "unittest")) return false;
+  if (check.targetQualifications !== undefined) {
+    return check.targetRunner === "pytest" && hasExactZeroTestObservations(check);
+  }
+  const targeted = checks.find((candidate) => candidate.id === `${check.id}:targeted`);
+  if (targeted === undefined || !hasExactZeroTestObservations(targeted)) return false;
+  return targeted.status === "passed"
+    || (targeted.status === "failed" && targeted.exitCode === 5 && targeted.targetRunner === "pytest");
 }
 
 function action(kind: EvidenceNextAction["kind"], detail: string, requiresRepositoryCodeExecution = false): EvidenceNextAction {
@@ -32,7 +48,7 @@ export function explainEvidenceBoundary(item: FileAssessment, checks: CheckResul
   const skippedObservation = exactObservations.find(({ observation }) => observation.outcome === "skipped");
   const unlocalizedTargetFailure = applicable.some((check) => check.targetQualifications !== undefined
     && check.status === "failed"
-    && !recognizedNoTestsExit(check)
+    && !recognizedNoTestsExit(check, applicable)
     && (
       check.targetObservations?.some((observation) => observation.outcome === "failed" && qualificationForObservation(check, observation)?.confidence === "high") !== true
       || check.targetObservations?.some((observation) => item.relatedTests.includes(observation.path) && observation.outcome === "not-observed" && qualificationForObservation(check, observation)?.confidence === "high") === true

tests/analyze.test.ts

@@ -327,6 +327,41 @@ test("pytest exit 5 remains a non-failing no-collection outcome", async (context
   assert.ok(!report.assessments[0]?.evidence.some((item) => item.kind === "failing-check"));
 });
 
+test("pytest exit 5 without lifecycle observations remains a verification failure", async (context) => {
+  const root = await initializeRepository({
+    "pyproject.toml": "[tool.pytest]\n",
+    "pytest.py": "raise SystemExit(5)\n",
+    "value.py": "def value():\n    return 1\n",
+    "tests/test_value.py": "from value import value\ndef test_value():\n    assert value() == 2\n",
+  });
+  context.after(() => rm(root, { recursive: true, force: true }));
+  await writeFiles(root, { "value.py": "def value():\n    return 2\n" });
+  const report = await analyzeRepository({ repo: root, runChecks: true, timeoutMs: 20_000 });
+  const pytestChecks = report.checks.filter((check) => check.targetRunner === "pytest" && check.exitCode === 5);
+  assert.ok(pytestChecks.length >= 1);
+  assert.ok(pytestChecks.some((check) => check.targetQualifications !== undefined && check.targetObservations?.[0]?.outcome === "not-observed"));
+  assert.equal(report.assessments[0]?.status, "verification-failed");
+  assert.ok(report.assessments[0]?.evidence.some((item) => item.kind === "failing-check"));
+  assert.equal(report.assessments[0]?.evidenceBoundary?.reason, "failure-unattributed");
+});
+
+test("unittest exit 5 requires corroborating zero-test observations instead of being blindly suppressed", async (context) => {
+  const root = await initializeRepository({
+    "unittest.py": "raise SystemExit(5)\n",
+    "value.py": "def value():\n    return 1\n",
+    "tests/test_value.py": "import unittest\nfrom value import value\nclass ValueTest(unittest.TestCase):\n    def test_value(self):\n        self.assertEqual(value(), 2)\n",
+  });
+  context.after(() => rm(root, { recursive: true, force: true }));
+  await writeFiles(root, { "value.py": "def value():\n    return 2\n" });
+  const report = await analyzeRepository({ repo: root, runChecks: true, timeoutMs: 20_000 });
+  const unittestChecks = report.checks.filter((check) => check.targetRunner === "unittest" && check.exitCode === 5);
+  assert.ok(unittestChecks.length >= 1);
+  assert.ok(unittestChecks.every((check) => check.status === "failed"));
+  assert.equal(report.assessments[0]?.status, "verification-failed");
+  assert.ok(report.assessments[0]?.evidence.some((item) => item.kind === "failing-check"));
+  assert.equal(report.assessments[0]?.evidenceBoundary?.reason, "failure-unattributed");
+});
+
 test("a unittest loader failure without an attributable test failure fails closed", async (context) => {
   const root = await initializeRepository({
     "value.py": "def value():\n    return 1\n",