Fix nightly workflow completion and merge evidence

.github/scripts/check-agent-result.cjs

@@ -0,0 +1,55 @@
+const { readFileSync } = require("node:fs");
+
+const terminalTypes = new Set(["create_pull_request", "noop", "report_incomplete"]);
+
+function requireTerminalResult(content) {
+  const results = [];
+  for (const [index, line] of content.split(/\r?\n/).entries()) {
+    if (!line.trim()) {
+      continue;
+    }
+
+    let item;
+    try {
+      item = JSON.parse(line);
+    } catch {
+      throw new Error(`Safe output line ${index + 1} is not valid JSON.`);
+    }
+    if (
+      item === null ||
+      typeof item !== "object" ||
+      Array.isArray(item) ||
+      typeof item.type !== "string"
+    ) {
+      throw new Error(`Safe output line ${index + 1} must be an object with a type.`);
+    }
+
+    const type = item.type.replaceAll("-", "_");
+    if (terminalTypes.has(type)) {
+      results.push(type);
+    }
+  }
+
+  if (results.length !== 1) {
+    throw new Error(
+      `Expected exactly one terminal safe output, found ${results.length}. ` +
+        "Call create_pull_request, noop, or report_incomplete; a prose-only explanation is not a result.",
+    );
+  }
+  return results[0];
+}
+
+if (require.main === module) {
+  try {
+    if (process.argv.length !== 3) {
+      throw new Error("Usage: node check-agent-result.cjs <safe-outputs.jsonl>");
+    }
+    const result = requireTerminalResult(readFileSync(process.argv[2], "utf8"));
+    console.log(`Verified terminal safe output: ${result}`);
+  } catch (error) {
+    console.error(`::error::${error instanceof Error ? error.message : String(error)}`);
+    process.exitCode = 1;
+  }
+}
+
+module.exports = { requireTerminalResult };

.github/scripts/check-agent-result.test.cjs

@@ -0,0 +1,127 @@
+const assert = require("node:assert/strict");
+const { spawnSync } = require("node:child_process");
+const { copyFileSync, mkdtempSync, rmdirSync, unlinkSync, writeFileSync } = require("node:fs");
+const { tmpdir } = require("node:os");
+const { join } = require("node:path");
+const { test } = require("node:test");
+const { requireTerminalResult } = require("./check-agent-result.cjs");
+
+const script = join(__dirname, "check-agent-result.cjs");
+const encode = (items) => items.map((item) => JSON.stringify(item)).join("\n");
+
+for (const item of [
+  { type: "noop", message: "Completed review; no eligible change." },
+  { type: "create_pull_request", title: "Fix behavior", body: "Verified fix.", branch: "fix-behavior" },
+  { type: "report_incomplete", reason: "Duplicate evidence is blocked.", details: "No edits published." },
+]) {
+  test(`accepts one ${item.type} result`, () => {
+    assert.equal(requireTerminalResult(encode([item])), item.type);
+  });
+}
+
+test("matches gh-aw's hyphenated type normalization", () => {
+  assert.equal(
+    requireTerminalResult(encode([{ type: "report-incomplete", reason: "Blocked." }])),
+    "report_incomplete",
+  );
+});
+
+test("allows other safe outputs alongside one terminal result", () => {
+  assert.equal(
+    requireTerminalResult(encode([{ type: "missing_data", reason: "Unavailable." }, { type: "report_incomplete" }])),
+    "report_incomplete",
+  );
+});
+
+test("accepts blank lines and CRLF output", () => {
+  assert.equal(requireTerminalResult('\r\n{"type":"noop"}\r\n\r\n'), "noop");
+});
+
+for (const content of ["", " \n\r\n", '{"type":"missing_data"}\n']) {
+  test(`rejects output without a terminal result: ${JSON.stringify(content)}`, () => {
+    assert.throws(() => requireTerminalResult(content), /exactly one terminal safe output, found 0/);
+  });
+}
+
+test("rejects the blocked-check prose instead of treating it as success", () => {
+  assert.throws(
+    () => requireTerminalResult("Unable to publish because the required duplicate check was incomplete."),
+    /not valid JSON/,
+  );
+});
+
+test("rejects contradictory terminal results", () => {
+  assert.throws(
+    () => requireTerminalResult(encode([{ type: "noop" }, { type: "report_incomplete" }])),
+    /exactly one terminal safe output, found 2/,
+  );
+});
+
+test("rejects repeated terminal calls", () => {
+  assert.throws(
+    () => requireTerminalResult(encode([{ type: "noop" }, { type: "noop" }])),
+    /exactly one terminal safe output, found 2/,
+  );
+});
+
+for (const item of [null, [], 1, "noop", {}, { type: 1 }]) {
+  test(`rejects invalid safe-output records: ${JSON.stringify(item)}`, () => {
+    assert.throws(() => requireTerminalResult(encode([item])), /object with a type/);
+  });
+}
+
+test("does not ignore malformed lines after a valid result", () => {
+  assert.throws(() => requireTerminalResult('{"type":"noop"}\ninvalid'), /line 2 is not valid JSON/);
+});
+
+test("CLI fails a missing result file with an error annotation", (t) => {
+  const directory = mkdtempSync(join(tmpdir(), "agent-result-"));
+  t.after(() => rmdirSync(directory));
+  const result = spawnSync(process.execPath, [script, join(directory, "missing.jsonl")], { encoding: "utf8" });
+  assert.equal(result.status, 1);
+  assert.match(result.stderr, /::error::.*ENOENT/);
+});
+
+test("prepared validation is unaffected by later checkout edits", (t) => {
+  const directory = mkdtempSync(join(tmpdir(), "agent-result-"));
+  const checkoutScript = join(directory, "checkout.cjs");
+  const preparedScript = join(directory, "prepared.cjs");
+  const output = join(directory, "safe-outputs.jsonl");
+  copyFileSync(script, checkoutScript);
+  copyFileSync(checkoutScript, preparedScript);
+  writeFileSync(checkoutScript, "process.exit(0);\n");
+  writeFileSync(output, "");
+  t.after(() => {
+    for (const file of [checkoutScript, preparedScript, output]) {
+      unlinkSync(file);
+    }
+    rmdirSync(directory);
+  });
+  const result = spawnSync(process.execPath, [preparedScript, output], { encoding: "utf8" });
+  assert.equal(result.status, 1);
+  assert.match(result.stderr, /::error::.*found 0/);
+});
+
+for (const [name, content, status] of [
+  ["empty result", "", 1],
+  ["completed review", '{"type":"noop","message":"Reviewed."}\n', 0],
+  ["reported blocker", '{"type":"report_incomplete","reason":"Evidence is blocked."}\n', 0],
+]) {
+  test(`CLI handles ${name}`, (t) => {
+    const directory = mkdtempSync(join(tmpdir(), "agent-result-"));
+    const file = join(directory, "safe-outputs.jsonl");
+    writeFileSync(file, content);
+    t.after(() => {
+      unlinkSync(file);
+      rmdirSync(directory);
+    });
+    const result = spawnSync(process.execPath, [script, file], { encoding: "utf8" });
+    assert.equal(result.status, status, result.stderr);
+    if (status === 0) {
+      assert.match(result.stdout, /Verified terminal safe output/);
+      assert.equal(result.stderr, "");
+    } else {
+      assert.match(result.stderr, /::error::.*found 0/);
+    }
+  });
+}

.github/workflows/dotnet-port-api-nightly.md

@@ -51,6 +51,14 @@ steps:
         mkdir -p "$data"
         go -C _catalog run ./cmd/symbolmap gaps -limit 0 > "$data/catalog-gaps.json"
         printf 'DOTNET_GAPS_FILE=%s\n' "$data/catalog-gaps.json" >> "$GITHUB_ENV"
+   - name: Prepare terminal result validation
+     run: cp .github/scripts/check-agent-result.cjs "$RUNNER_TEMP/gh-aw/check-agent-result.cjs"
+post-steps:
+   - name: Require a terminal agent result
+     if: success()
+     env:
+        GH_AW_SAFE_OUTPUTS: ${{ steps.set-runtime-paths.outputs.GH_AW_SAFE_OUTPUTS }}
+     run: node "$RUNNER_TEMP/gh-aw/check-agent-result.cjs" "$GH_AW_SAFE_OUTPUTS"
 permissions:
    contents: read
    pull-requests: read
@@ -144,8 +152,12 @@ For `search_issues` and `search_pull_requests`, always include `repo:microsoft/a
 
 Searches discover references; direct reads resolve them. Track references by repository, resource type (`issue` or `pr`), and number, with their candidate/query and any filtering or failure. Read relevant bodies, comments, diffs, and outcomes directly through the matching tool; never include bodies in search results or re-search to rediscover a known reference. Reuse resolved evidence across candidate checks. Skip only work that implements, actively claims, or explicitly rejects the exact missing behavior, including older fallback tracking issues. A matching title, base feature port, internal refactor, or preservation test is not enough; a closed PR is not necessarily merged.
 
+Current Go code is the source of truth for implemented coverage. Identify the actual declarations, callers, and tests that satisfy the selected contract; a proposed patch or a PR body's description is not checkout evidence. When citing a PR as implemented work, use `pull_request_read` with `method: get` and record its `state`, `merged`, and `merged_at`. Require `merged: true` and a non-null merge timestamp, then verify the behavior still exists in the current checkout. Search/list results and `state: closed` alone do not establish a merge. A closed, unmerged PR may document a maintainer's explicit rejection, but record that as a rejected proposal, never as implemented coverage; without such a decision, reassess the candidate against current code.
+
 A filtered search item is not an explicit denial of a direct read. For each potentially relevant filtered reference, attempt one matching `issue_read` or `pull_request_read` through the same guarded GitHub MCP, unless that direct read was already denied. If allowed, assess its actual relevance; if denied or required evidence remains unavailable, keep that candidate blocked during selection and final recheck. Do not retry an explicitly denied read, lower integrity policy, change transports, call a trapped guard, or use sandbox `gh` or unconfigured downloads. Narrower empty searches, an accessible partial diff, passing tests, or disclosing the filter do not clear unresolved evidence.
 
+Filtering that hides issue/PR identifiers is also unresolved evidence, not an empty result. Record the query and filtered count without inventing references or searching around the policy. If required duplicate evidence still blocks an otherwise eligible candidate and no fully verified candidate can be selected within the budget, call `report_incomplete`; do not stop with a prose-only explanation or substitute `noop`. Source-backed exclusions remain valid, and a blocked alternative does not invalidate a fully verified selection.
+
 ## Review and selection
 
 Verify `DOTNET_UPSTREAM_SHA` and screen every row of `DOTNET_GAPS_FILE`, reading remaining ranges if a preview is cut off. Record the screened count and `page.total`; an unreadable or incompletely screened report is incomplete selection, not proof of no work. Keep selection read-only: group related catalog leaves into coherent candidates and inspect at most three promising groups in depth.
@@ -168,6 +180,8 @@ Before editing, confirm the selected catalog leaves, the pinned .NET contract an
 
 Call one terminal safe-output tool and inspect its result before final prose; a prose summary is not a tool call:
 
+Post-execution validation requires exactly one `create_pull_request`, `noop`, or `report_incomplete` result. Exit code zero does not waive this requirement. For every claim of already-implemented coverage, include current Go file/symbol evidence and the verified merge status of any PR cited as implementation evidence.
+
 - `create_pull_request`: one verified, tested gap closure with a clear final duplicate check. Send only `title`, `body`, and `branch`; omit `temporary_id` and other optional arguments. Draft status, base branch, and title prefix are configured already; this single PR needs no temporary ID. Use a concrete title and sections **Summary**, **Ported .NET PRs**, **Breaking Changes**, **Tests and Examples**, and **Notes**. Include inventoried package versions, pinned SHA, exact catalog leaves and expected assessment changes, relevant upstream commits/PRs (or `None` if no specific PR was ported), immutable evidence, experimental and duplicate checks, and actual validation. Keep prose concise without hard-wrapping paragraphs. Report publication as queued, not a confirmed PR; never push, merge, approve, or open PRs directly.
 - `noop`: completed bounded gap review with no eligible unclaimed change and no potentially eligible candidate left blocked. Source-backed exclusions, including experimental, fix-only, already covered, and too broad, are completed reviews, not blockers. State the pinned SHA, screened/total gap counts, actual leaves/groups inspected, exclusion or already-satisfied reasons, and existing-work links. Do not claim a full catalog audit from screening rows and inspecting a few candidates; an empty diff alone does not establish completion.
 - `report_incomplete`: no fully verified selection is possible because required selection/source/duplicate evidence remains unresolved, or the selected port's validation fails, or execution reaches a runtime limit. Send only `reason` and `details`. Include the operation/error, recovery attempts, SHA, screened/total gap counts, blocked candidates, and remaining work. Leave partial edits unpublished; do not substitute `noop` or report a working-but-denied tool as missing. The three-group selection limit alone is not a failure.