Fail closed on unattributed batch failures

CHANGELOG.md

@@ -6,6 +6,7 @@ All notable changes are documented here. This project follows Semantic Versionin
 
 ### Fixed
 
+- Failed closed on unattributed targeted-runner process failures even when another qualified batch target is unavailable. An unavailable target can no longer act as a sink that lets unrelated observed passes survive an ambiguous process-level failure; only explicitly failed targets retain localized failure attribution.
 - Failed closed on deletion-only zero-context hunks when attributing current changed symbols and call references. Git represents a deletion-only hunk with a zero-length new-side range anchored to a neighboring current line; ProofDiff now reconciles reconstructed current-line spans with Git numstat additions before treating those anchors as changed current code.
 - Preserved precise changed-line hunks across renames by scoping per-file Git diffs to both the previous and current path. This prevents a rename with content edits from being reinterpreted as a whole-file addition and overstating changed-line or call-reference evidence.
 - Preserved trailing whitespace in Git repository-root paths. `findRepository()` now removes only Git's record line terminator instead of applying JavaScript `String.trim()`, preventing a repository such as `repository ` from being silently redirected to a sibling path with the trimmed spelling.

src/checks.ts

@@ -474,11 +474,10 @@ export function parseTargetObservations(root: string, check: CheckDefinition, ra
     records.set(absolute, record);
   }
   if (records.size !== expected.size) return notObserved(qualifications, "The runner observation omitted one or more qualified targets and was rejected.");
-  const unavailableRecords = [...records.values()].filter((record) => !record.observed);
-  const processFailureHasNoUnavailableTarget = payload.unattributedFailures > 0 && unavailableRecords.length === 0;
+  const hasUnattributedProcessFailure = payload.unattributedFailures > 0;
   return qualifications.map((qualification) => {
     const record = records.get(path.resolve(root, qualification.runnerPath))!;
-    if (!record.observed || (processFailureHasNoUnavailableTarget && record.failed === 0)) {
+    if (!record.observed || (hasUnattributedProcessFailure && record.failed === 0)) {
       const detail = !record.observed
         ? "The runner did not produce a trustworthy lifecycle observation for this exact target."
         : "The runner reported an unattributed process-level failure that could not be excluded from this target.";

tests/analyze.test.ts

@@ -142,6 +142,40 @@ test("Python unittest repositories receive AST-backed related evidence", async (
   assert.equal(report.assessments[0]?.status, "verified");
 });
 
+test("unattributed unittest failures cannot hide behind an unavailable batch target", async (context) => {
+  const root = await initializeRepository({
+    "src/__init__.py": "",
+    "src/a.py": "def value_a():\n    return 1\n",
+    "src/b.py": "def value_b():\n    return 1\n",
+    "src/c.py": "def value_c():\n    return 1\n",
+    "tests/__init__.py": "",
+    "tests/helper.py": "import unittest\n\nclass ImportedFailure(unittest.TestCase):\n    def test_imported_failure(self):\n        self.fail('imported failure')\n",
+    "tests/test_a.py": "import unittest\nfrom src.a import value_a\nfrom tests.helper import ImportedFailure\n\nclass ATest(unittest.TestCase):\n    def test_a(self):\n        self.assertEqual(value_a(), 2)\n",
+    "tests/test_b.py": "import unittest\nfrom src.b import value_b\n\nclass BTest(unittest.TestCase):\n    def test_b(self):\n        self.assertEqual(value_b(), 999)\n",
+    "tests/test_c.py": "import unittest\nfrom src.c import value_c\n\nvalue_c()\n",
+  });
+  context.after(() => rm(root, { recursive: true, force: true }));
+  await writeFiles(root, {
+    "src/a.py": "def value_a():\n    return 2\n",
+    "src/b.py": "def value_b():\n    return 2\n",
+    "src/c.py": "def value_c():\n    return 2\n",
+  });
+
+  const report = await analyzeRepository({ repo: root, runChecks: true, timeoutMs: 20_000 });
+  const targeted = report.checks.find((check) => check.id === "python:test:unittest:targeted");
+  const observations = new Map(targeted?.targetObservations?.map((observation) => [observation.path, observation]));
+  const sourceA = report.assessments.find((assessment) => assessment.file.path === "src/a.py");
+
+  assert.equal(targeted?.status, "failed", targeted?.output);
+  assert.equal(observations.get("tests/test_a.py")?.outcome, "not-observed");
+  assert.match(observations.get("tests/test_a.py")?.detail ?? "", /unattributed process-level failure/);
+  assert.equal(observations.get("tests/test_b.py")?.outcome, "failed");
+  assert.equal(observations.get("tests/test_c.py")?.outcome, "not-observed");
+  assert.equal(sourceA?.status, "verification-failed");
+  assert.deepEqual(sourceA?.executedTests, []);
+  assert.deepEqual(sourceA?.testExecutions, []);
+});
+
 test("Python stubs remain static test relationships without inventing a runnable framework", async (context) => {
   const root = await initializeRepository({
     "value.py": "def value():\n    return 1\n",
@@ -448,7 +482,7 @@ test("a partially localized targeted batch fails closed for an ambiguous related
   assert.deepEqual(targeted?.targetObservations?.map((item) => [item.path, item.outcome]), [
     ["tests/test_fail.py", "failed"],
     ["tests/test_import.py", "not-observed"],
-    ["tests/test_pass.py", "passed"],
+    ["tests/test_pass.py", "not-observed"],
   ]);
   assert.equal(assessment?.status, "verification-failed");
   assert.ok(assessment?.evidence.some((item) => item.kind === "failing-check" && item.checkId === targeted.id));
@@ -460,7 +494,8 @@ test("a partially localized targeted batch fails closed for an ambiguous related
   const valueSummary = renderGithubSummary(valueReport);
   assert.match(valueSummary, /failed without complete attribution/);
   assert.match(valueSummary, /tests\/test_import\.py: not-observed/);
-  assert.match(valueSummary, /Independently passing target: <code>tests\/test_pass\.py<\/code>/);
+  assert.match(valueSummary, /tests\/test_pass\.py: not-observed/);
+  assert.doesNotMatch(valueSummary, /Independently passing target: <code>tests\/test_pass\.py<\/code>/);
   assert.doesNotMatch(valueSummary, /Attributed failed target: <code>tests\/test_pass\.py/);
 });