diff --git a/.github/workflows/repair-cluster-worker.yml b/.github/workflows/repair-cluster-worker.yml index 24b287b690..b546a046da 100644 --- a/.github/workflows/repair-cluster-worker.yml +++ b/.github/workflows/repair-cluster-worker.yml @@ -892,3 +892,17 @@ jobs: if: ${{ always() && failure() && steps.crabfleet_session.outcome == 'success' && (steps.repair_requeue.outputs.count == '' || steps.repair_requeue.outputs.count == '0' || steps.requeue_dispatch.outcome != 'success') }} continue-on-error: true run: pnpm run repair:action-session -- update --state blocked --phase action_failed --summary "Repair Action failed before all completion gates passed" --completion-reason action_failed + + - name: Reconcile failed automerge telemetry + if: ${{ always() && failure() && inputs.automerge_session_id != '' && (steps.repair_requeue.outputs.count == '' || steps.repair_requeue.outputs.count == '0' || steps.requeue_dispatch.outcome != 'success') }} + continue-on-error: true + env: + AUTOMERGE_SESSION_ID: ${{ inputs.automerge_session_id }} + AUTOMERGE_RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + CLAWSWEEPER_STATUS_INGEST_TOKEN: ${{ secrets.CLAWSWEEPER_STATUS_INGEST_TOKEN }} + GITHUB_TOKEN: ${{ github.token }} + run: | + node scripts/dashboard-reconcile-automerge.ts \ + --session-id "$AUTOMERGE_SESSION_ID" \ + --run-url "$AUTOMERGE_RUN_URL" \ + --run-conclusion failure diff --git a/dashboard/automerge-metrics.ts b/dashboard/automerge-metrics.ts index 9e4ac1a63f..a7ee7c6656 100644 --- a/dashboard/automerge-metrics.ts +++ b/dashboard/automerge-metrics.ts @@ -114,6 +114,7 @@ export function summarizeAutomergeMetrics( range?: string; repo?: string | null; policyVersion?: string | null; + sessionId?: string | null; now?: string; activeOnly?: boolean; sessionLimit?: number; @@ -131,7 +132,8 @@ export function summarizeAutomergeMetrics( const filtered = allEvents.filter( (event) => (!options.repo || event.repository === options.repo) && - (!options.policyVersion || event.policy_version === options.policyVersion), + (!options.policyVersion || event.policy_version === options.policyVersion) && + (!options.sessionId || event.session_id === options.sessionId), ); const sessions = projectSessions(filtered); const terminal = sessions.filter( @@ -192,7 +194,7 @@ export function summarizeAutomergeMetrics( return options.activeOnly ? left - right : right - left; }) .slice(0, sessionLimit); - const filtersActive = Boolean(options.repo || options.policyVersion); + const filtersActive = Boolean(options.repo || options.policyVersion || options.sessionId); const telemetrySince = filtersActive ? (filtered[0]?.occurred_at ?? null) : ((ledger as AutomergeMetricLedger | null)?.telemetry_since ?? diff --git a/dashboard/worker.ts b/dashboard/worker.ts index ba88af3847..eaaf2109ec 100644 --- a/dashboard/worker.ts +++ b/dashboard/worker.ts @@ -893,6 +893,7 @@ async function automergeMetricsJson(request, env) { range: url.searchParams.get("range") ?? "7d", repo: url.searchParams.get("repo"), policyVersion: url.searchParams.get("policy_version"), + sessionId: url.searchParams.get("session_id"), activeOnly: url.searchParams.get("active_only") === "true", sessionLimit: Number(url.searchParams.get("session_limit")), }), @@ -9352,11 +9353,11 @@ function renderAutomergeProduct(data) { return '
n=' + fmt.format(bucket.merged_count) + '
'; }).join(""); const chart = '
' + (activeAutomergeChart === "success" ? '● normal sample · ○ fewer than 5 terminal sessions · gaps mean no terminal sample' : '● p50 · amber p90 · gaps mean no merged sample') + '
'; - const outcomeLabels = { merged: "Merged", maintainer_stopped: "Maintainer stopped", repair_cap_exhausted: "Repair cap exhausted", pr_closed: "PR closed", automerge_disabled: "Automerge disabled" }; + const outcomeLabels = { merged: "Merged", repair_failed: "Repair workflow failed", maintainer_stopped: "Maintainer stopped", repair_cap_exhausted: "Repair cap exhausted", pr_closed: "PR closed", automerge_disabled: "Automerge disabled" }; const outcomes = Object.entries(outcomeLabels).map(entry => '
' + esc(entry[1]) + '' + fmt.format(data.terminal_outcomes?.[entry[0]] || 0) + '
').join(""); const efficiency = [['0 base sync sessions', data.repair_efficiency?.zero_base_sync], ['1 base sync session', data.repair_efficiency?.one_base_sync], ['2+ base sync sessions', data.repair_efficiency?.multiple_base_sync], ['Multi-rebase rate', value(summary.multi_rebase_rate_percent, "%")]].map(entry => '
' + esc(entry[0]) + '' + esc(entry[1] ?? 0) + '
').join(""); const details = '

Terminal outcomes

' + outcomes + '

Repair efficiency

' + efficiency + '
'; - const rows = (data.sessions || []).map(session => '' + linkClass(session.pr_url, session.repository + '#' + session.item_number, "item-link") + '' + esc(session.state || 'unknown') + '' + esc(session.policy_version) + '' + esc(session.activated_at ? since(session.activated_at) : 'missing') + '' + esc(session.terminal_at ? since(session.terminal_at) : since(session.last_event_at)) + '' + fmt.format(session.base_sync_count || 0) + '' + fmt.format(session.repairs || 0) + '' + esc(session.last_reason || '') + ' ' + linkClass(session.run_url, 'run', 'pill run-link') + '').join(""); + const rows = (data.sessions || []).map(session => '' + linkClass(session.pr_url, session.repository + '#' + session.item_number, "item-link") + '' + esc(outcomeLabels[session.state] || session.state || 'unknown') + '' + esc(session.policy_version) + '' + esc(session.activated_at ? since(session.activated_at) : 'missing') + '' + esc(session.terminal_at ? since(session.terminal_at) : since(session.last_event_at)) + '' + fmt.format(session.base_sync_count || 0) + '' + fmt.format(session.repairs || 0) + '' + esc(session.last_reason || '') + ' ' + linkClass(session.run_url, 'run', 'pill run-link') + '').join(""); const sessions = '

Recent automerge sessions

Showing up to 30 latest sessions in the selected window
' + (rows || '') + '
PRStatePolicyActivatedTerminal / ageSyncsRepairsLast reason
No session telemetry in this range.
'; document.getElementById("automerge-product").innerHTML = kpis + chart + details + sessions; } diff --git a/docs/automerge-optimization-plan.md b/docs/automerge-optimization-plan.md new file mode 100644 index 0000000000..f566ac0522 --- /dev/null +++ b/docs/automerge-optimization-plan.md @@ -0,0 +1,159 @@ +# Automerge Reliability Improvement Plan + +## Purpose + +Make ClawSweeper automerge predictable, recoverable, and safe by moving from +production-discovered failures to local, replayable evidence. This plan is a +sequence: do not treat an apparent workflow success as proof that an automerge +attempt reached a safe product terminal state. + +## Scope and safety boundary + +- All early validation uses fixed revisions, local bare Git remotes, and a + fail-closed GitHub simulator. +- It never writes GitHub comments, labels, branches, merges, or closes items. +- It does not change production automerge behavior during the stable-red + phase. Production fixes begin only after a reproducer is accepted. +- Local tests must use explicit process, tree, and state oracles rather than + treating a command exit code as the complete result. + +## Phases + +### 1. Establish stable red evidence + +Build a proof-producing E2E harness around production artifacts. Each proof +binds the candidate revision and executable/dependency digests, fixture digest, +event sequence, fault point, phase, product outcome, Git tree state, and child +process observation. + +The gate has two deliberate modes: + +- `candidate`: a known product violation is red and exits non-zero. +- `reproducer`: the same exact fingerprint is a confirmed historical + reproducer and exits zero. + +The initial evidence set includes OpenClaw runtime incidents, state-publication +tree-loss cases, immutable canonical paths, duplicate delivery/replay, +replacement runs, modeled crashes, and mutation-sensitive head/base/check/ +review/permission/label drift. + +Stable red requires deterministic inputs and scheduling, the same phase and +normalized fingerprint on every run, no timeout/network/race substitute +failure, ten of ten fast repetitions, and three of three real OpenClaw +repetitions. A model-only result is evidence, not a release-candidate pass. + +### 2. Repair one proved invariant at a time + +For each accepted red proof, make the narrowest production change that restores +the violated invariant. Preserve the reproducer, add the green counterpart, +and require candidate mode to pass without weakening the oracle. Do not hide a +failure with retries, larger timeouts, skips, or a broader fixture. + +### 3. Validate the GitHub contract minimally + +After local evidence is green, validate only the platform-specific contract +that cannot be modeled locally: Actions delivery and cancellation behavior, +GitHub permission/ruleset combinations, merge-queue semantics, and API +eventual consistency. Keep the validation proposal-only until its scope is +explicitly authorized. + +### 4. Refactor toward durable state ownership + +Move the automerge path toward explicit durable intent, attempt, outcome, and +publication records. Reconciliation must be idempotent: repeated commands, +deliveries, and restarts may observe or finish an existing fact but must not +create a second logical merge, comment, or publication. + +Split responsibilities into small modules: + +- admission and immutable job intent; +- exact-head/readiness verdicts; +- execution and containment; +- durable result/outcome recording; +- reconciliation and router recovery; +- state-tree publication with CAS/tree-preservation checks; +- proof and operational reporting. + +The state repository remains the durable status surface. A missing, stale, or +conflicting fact blocks mutation rather than selecting a best-effort path. + +## Required invariants + +- Exact head/base, checks, review, permission, and protected-label facts are + revalidated immediately before a merge mutation. +- A workflow's successful process exit cannot override leaked descendants or a + blocked product state. +- State publication preserves concurrent siblings, merge parents, and existing + immutable entries; a conflicting canonical path fails closed. +- A crash after durable intent is recoverable without duplicate work; a crash + after merge records one outcome without a second merge. +- Candidate, fixture, dependency, and proof identity drift are harness errors. + +## Exit criteria + +The stable-red phase is complete only when all required incident and topology +scenarios have deterministic proofs, fast and real repetition inventories are +complete, controls pass, the full local gate has a measured hot-cache budget, +and no local test touches live GitHub state. The handoff is a failure inventory +and replayable proof bundle, not a production fix. + +## Current status + +This is a handoff record. The stable-red phase is **not complete** and no +production Automerge behavior has been changed. + +### Completed and evidence-backed + +- The CLI supports explicit flow, publication, model, and runtime scenario + selection; candidate and reproducer exit semantics are distinct and + fail-closed. +- Proof summaries bind the candidate `HEAD`, production `dist/` digest, + dependency digest, fixture identity, event/fault sequence, terminal state, + and child-process observation. Candidate identity is rechecked after every + scenario. +- The real OpenClaw git-hooks incident is confirmed against pinned revisions: + `openclaw.unsafe-core-hookspath` at `target-setup-git-safety`. +- Both historical publication P1 cases are confirmed 10/10 against + `5c28770b`: `state-publication.concurrent-sibling-lost` and + `state-publication.merge-tree-entry-lost` at `state-publication-rebuild`. +- The immutable canonical-path conflict is safe 10/10: different bytes are + blocked and the original remote tree entry is preserved. +- Pending-run replacement, duplicate command replay, both crash models, and + all modeled mutation-sensitive drift cases are deterministic 10/10. They + intentionally remain `evidence-only`, not a candidate-release pass. +- Focused proof/candidate/gate/model tests (23), syntax checks, formatting, and + diff checks passed on Node 24. + +### Incomplete or explicitly blocked + +- The real process-leak case is **not confirmed**. With the pinned target tree, + the real `pnpm check:changed` exits at the max-lines ratchet before the + historical "exit 0 with four descendants" oracle. Its proof is correctly a + `harness-error`; do not skip the ratchet or relabel this as confirmed. +- The flow 10/10 run completed but its inventory is `unstable`: flow fixture + digests currently include per-run temporary Git metadata. Fix the flow + fixture-digest contract before accepting repetition evidence; do not simply + remove fixture identity from the stability signature. +- The full Node 24 `pnpm run check` was started in the shared workspace while + other independent checks were also active. Re-run it in a quiet workspace and + record the complete result before declaring the phase ready for repair. + +### Next owner checklist + +1. Reconstruct the historical process-leak workflow context from read-only + evidence and explain the ratchet discrepancy while retaining real setup and + `check:changed`; then require 3/3 warmup/final fingerprint agreement. +2. Make the flow fixture digest represent stable fixture content rather than + generated commit metadata, add a focused regression test, and rerun flow + 10/10. A stable signature must still bind the actual fixture inputs. +3. Preserve the existing publication/model inventories, then compose a gate + inventory by fixed candidate revision rather than pretending one candidate + covers incompatible historical revisions. +4. Run the complete Node 24 check in isolation. Do not repair unrelated apply + tests as part of this E2E-only phase. +5. Only after every stable-red exit criterion has evidence may the next PR + begin a narrow production repair; retain every red reproducer as its oracle. + +Useful local proof roots from the latest run are `/tmp/clawsweeper-e2e-closeout-*`. +They are disposable evidence, not repository fixtures; regenerate them rather +than relying on their lifetime. diff --git a/scripts/dashboard-reconcile-automerge.ts b/scripts/dashboard-reconcile-automerge.ts index a6369df0bf..0df678807a 100644 --- a/scripts/dashboard-reconcile-automerge.ts +++ b/scripts/dashboard-reconcile-automerge.ts @@ -7,7 +7,11 @@ const DEFAULT_LIMIT = 8; const MAX_LIMIT = 20; const DEFAULT_TIMEOUT_MS = 15_000; const MAX_TIMEOUT_MS = 30_000; -const LOOKBACK_MS = 24 * 60 * 60 * 1000; +const LOOKBACK_MS = 7 * 24 * 60 * 60 * 1000; +const REPAIR_WORKFLOW = "repair-cluster-worker.yml"; +const REPAIR_REQUEUE_DISPATCH_STEP = "Requeue source-head repair races"; +const DEFAULT_WORKFLOW_REPOSITORY = "openclaw/clawsweeper"; +const REPAIR_FAILURE_CONCLUSIONS = new Set(["failure", "timed_out", "action_required"]); type ActiveSession = { session_id: string; @@ -24,44 +28,86 @@ type ReconcileOptions = { env?: NodeJS.ProcessEnv; fetcher?: typeof fetch; now?: string; + sessionId?: string | null; + runUrl?: string | null; + runConclusion?: string | null; +}; + +type Terminal = { + outcome: "merged" | "pr_closed" | "repair_failed"; + occurredAt: string; + reason: string; + runUrl?: string | null; }; export async function reconcileAutomergeProductMetrics({ env = process.env, fetcher = fetch, now = new Date().toISOString(), + sessionId = null, + runUrl = null, + runConclusion = null, }: ReconcileOptions = {}) { + const normalizedSessionId = nullableText(sessionId); + const normalizedRunUrl = nullableText(runUrl); + const normalizedRunConclusion = nullableText(runConclusion)?.toLowerCase() ?? null; + if (normalizedSessionId && !parseSessionId(normalizedSessionId)) { + return skippedResult("invalid_session_id", 0); + } + if (normalizedRunUrl && !parseRunUrl(normalizedRunUrl)) { + return skippedResult("invalid_run_url", 0); + } + if (normalizedRunConclusion && !REPAIR_FAILURE_CONCLUSIONS.has(normalizedRunConclusion)) { + return skippedResult("invalid_run_conclusion", 0); + } + const ingestToken = String(env.CLAWSWEEPER_STATUS_INGEST_TOKEN ?? "").trim(); - if (!ingestToken) return { ok: true, skipped: "ingest_token_missing", candidates: 0 }; + if (!ingestToken) return skippedResult("ingest_token_missing", 0); const statusUrl = trimTrailingSlash( env.CLAWSWEEPER_STATUS_URL || "https://clawsweeper.openclaw.ai", ); const ingestUrl = env.CLAWSWEEPER_STATUS_INGEST_URL || `${statusUrl}/api/events`; const githubToken = env.GITHUB_TOKEN || env.GH_TOKEN || ""; - const limit = boundedInt(env.CLAWSWEEPER_AUTOMERGE_RECONCILE_LIMIT, DEFAULT_LIMIT, MAX_LIMIT); + const workflowRepository = + nullableText(env.CLAWSWEEPER_AUTOMERGE_WORKFLOW_REPOSITORY) ?? DEFAULT_WORKFLOW_REPOSITORY; + if (!repositoryName(workflowRepository)) return skippedResult("invalid_workflow_repository", 0); + const limit = normalizedSessionId + ? 1 + : boundedInt(env.CLAWSWEEPER_AUTOMERGE_RECONCILE_LIMIT, DEFAULT_LIMIT, MAX_LIMIT); const timeoutMs = boundedInt( env.CLAWSWEEPER_AUTOMERGE_RECONCILE_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS, ); const nowMs = Date.parse(now); + if (!Number.isFinite(nowMs)) return skippedResult("invalid_now", 0); const deadline = AbortSignal.timeout(timeoutMs); try { const metricsUrl = new URL("/api/automerge-metrics", `${statusUrl}/`); - metricsUrl.searchParams.set("range", "24h"); + metricsUrl.searchParams.set("range", "7d"); metricsUrl.searchParams.set("active_only", "true"); metricsUrl.searchParams.set("session_limit", String(limit)); + if (normalizedSessionId) metricsUrl.searchParams.set("session_id", normalizedSessionId); const metrics = await fetchJson(metricsUrl, fetcher, deadline); - const candidates = activeSessions(metrics, nowMs).slice(0, limit); + const candidates = activeSessions(metrics, nowMs) + .filter((session) => !normalizedSessionId || session.session_id === normalizedSessionId) + .slice(0, limit); + if (normalizedSessionId && candidates.length === 0) { + return skippedResult("session_not_active", 0); + } - // Durable active sessions already form the retry queue. Re-reading authoritative - // PR state closes both past and future delivery gaps without a second outbox. + // Durable active sessions form the retry queue. PR state remains authoritative; + // only an open PR permits the lower-priority repair-run failure observation. const results = await Promise.all( candidates.map((session) => reconcileSession({ session, + runUrlOverride: normalizedRunUrl, + runConclusionOverride: normalizedRunConclusion, + observedAt: new Date(nowMs).toISOString(), + workflowRepository, ingestUrl, ingestToken, githubToken, @@ -73,15 +119,21 @@ export async function reconcileAutomergeProductMetrics({ return { ok: true, candidates: candidates.length, - github_reads: results.filter((result) => result.githubRead).length, + github_reads: results.reduce((total, result) => total + result.githubReads, 0), terminal: results.filter((result) => result.terminal).length, delivered: results.filter((result) => result.delivered).length, failed: results.filter((result) => result.error).length, + skipped: results.filter((result) => result.skipped).length, + results, }; } catch (error) { return { ok: false, candidates: 0, + github_reads: 0, + terminal: 0, + delivered: 0, + failed: 1, error: errorMessage(error), }; } @@ -89,6 +141,10 @@ export async function reconcileAutomergeProductMetrics({ async function reconcileSession({ session, + runUrlOverride, + runConclusionOverride, + observedAt, + workflowRepository, ingestUrl, ingestToken, githubToken, @@ -96,52 +152,98 @@ async function reconcileSession({ signal, }: { session: ActiveSession; + runUrlOverride: string | null; + runConclusionOverride: string | null; + observedAt: string; + workflowRepository: string; ingestUrl: string; ingestToken: string; githubToken: string; fetcher: typeof fetch; signal: AbortSignal; }) { + let githubReads = 0; try { const pr = await fetchJson( - new URL( - `/repos/${encodeURIComponent(session.repository.split("/")[0]!)}/${encodeURIComponent( - session.repository.split("/")[1]!, - )}/pulls/${session.item_number}`, - "https://api.github.com", - ), + githubApiUrl(session.repository, `/pulls/${session.item_number}`), fetcher, signal, - { - Accept: "application/vnd.github+json", - "X-GitHub-Api-Version": "2022-11-28", - "User-Agent": "openclaw-clawsweeper-automerge-reconciler", - ...(githubToken ? { Authorization: `Bearer ${githubToken}` } : {}), - }, + githubHeaders(githubToken), ); - const terminal = authoritativeTerminal(pr); - if (!terminal) return { githubRead: true, terminal: false, delivered: false }; - const event = terminalEvent(session, terminal); + githubReads += 1; + let terminal = authoritativePrTerminal(pr); + if (!terminal) { + const associatedRunUrl = runUrlOverride ?? nullableText(session.run_url); + const runIdentity = associatedRunUrl ? parseRunUrl(associatedRunUrl) : null; + if (!runIdentity || runIdentity.repository !== workflowRepository) { + return sessionSkip( + session, + githubReads, + runIdentity ? "external_run_repository" : "run_url_missing_or_invalid", + ); + } + const run = await fetchJson( + githubApiUrl(runIdentity.repository, `/actions/runs/${runIdentity.runId}`), + fetcher, + signal, + githubHeaders(githubToken), + ); + githubReads += 1; + terminal = authoritativeRepairFailure( + run, + runIdentity, + workflowRepository, + runConclusionOverride, + observedAt, + ); + if (!terminal) return sessionSkip(session, githubReads, "repair_run_not_failed"); + if (!runConclusionOverride) { + // repair-cluster-worker dispatches one non-matrix execute job for one + // automerge_session_id. Its receipt/cluster/execute jobs fit one 100-row + // page, so this run-wide step is session-specific. Revisit this contract + // before adding a matrix or multiple automerge sessions to that workflow. + // The matched step is requeue_dispatch, not the preceding count detector; + // its workflow condition requires a nonzero count before it can succeed. + const jobs = await fetchJson( + githubApiUrl( + runIdentity.repository, + `/actions/runs/${runIdentity.runId}/jobs?filter=latest&per_page=100`, + ), + fetcher, + signal, + githubHeaders(githubToken), + ); + githubReads += 1; + const requeued = repairRunSuccessfullyRequeued(jobs); + if (requeued === null) return sessionSkip(session, githubReads, "repair_run_jobs_invalid"); + if (requeued) return sessionSkip(session, githubReads, "repair_run_successfully_requeued"); + } + } + const response = await fetcher(ingestUrl, { method: "POST", headers: { Authorization: `Bearer ${ingestToken}`, "Content-Type": "application/json", }, - body: JSON.stringify(event), + body: JSON.stringify(terminalEvent(session, terminal)), signal, }); return { - githubRead: true, + session_id: session.session_id, + githubReads, terminal: true, delivered: response.ok, + skipped: null, error: response.ok ? null : `ingest returned ${response.status}`, }; } catch (error) { return { - githubRead: true, + session_id: session.session_id, + githubReads, terminal: false, delivered: false, + skipped: "github_or_network_error", error: errorMessage(error), }; } @@ -155,13 +257,12 @@ function activeSessions(value: unknown, nowMs: number): ActiveSession[] { return sessions.filter((value): value is ActiveSession => { if (!value || typeof value !== "object") return false; const session = value as Partial; + const identity = parseSessionId(String(session.session_id ?? "")); const lastEventAt = Date.parse(String(session.last_event_at ?? "")); return ( !session.terminal_at && - typeof session.session_id === "string" && - /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(String(session.repository ?? "")) && - Number.isInteger(session.item_number) && - Number(session.item_number) > 0 && + identity?.repository === session.repository && + identity?.itemNumber === session.item_number && Number.isFinite(lastEventAt) && lastEventAt >= nowMs - LOOKBACK_MS && lastEventAt <= nowMs @@ -169,25 +270,76 @@ function activeSessions(value: unknown, nowMs: number): ActiveSession[] { }); } -function authoritativeTerminal(value: unknown) { +function authoritativePrTerminal(value: unknown): Terminal | null { if (!value || typeof value !== "object") return null; const pr = value as { merged_at?: unknown; closed_at?: unknown; state?: unknown }; const mergedAt = isoTimestamp(pr.merged_at); - if (mergedAt) return { outcome: "merged", occurredAt: mergedAt } as const; + if (mergedAt) { + return { + outcome: "merged", + occurredAt: mergedAt, + reason: "reconciled from authoritative GitHub PR merged_at", + }; + } const closedAt = isoTimestamp(pr.closed_at); if (String(pr.state ?? "").toLowerCase() === "closed" && closedAt) { - return { outcome: "pr_closed", occurredAt: closedAt } as const; + return { + outcome: "pr_closed", + occurredAt: closedAt, + reason: "reconciled from authoritative GitHub PR closed_at", + }; } return null; } -function terminalEvent( - session: ActiveSession, - terminal: { outcome: "merged" | "pr_closed"; occurredAt: string }, -) { +function authoritativeRepairFailure( + value: unknown, + identity: { repository: string; runId: number; url: string }, + workflowRepository: string, + reportedConclusion: string | null, + observedAt: string, +): Terminal | null { + if (!value || typeof value !== "object") return null; + const run = value as Record; + const repository = repositoryName((run.repository as Record | null)?.full_name); + const workflowPath = String(run.path ?? "").split("@")[0]; + const apiConclusion = String(run.conclusion ?? "").toLowerCase(); + const status = String(run.status ?? "").toLowerCase(); + // The workflow knows an earlier step has irreversibly failed while GitHub still + // reports the current run as in progress. Once GitHub exposes a terminal result, + // that authoritative conclusion must win over the early best-effort report. + const useReportedConclusion = + status === "in_progress" && !apiConclusion && reportedConclusion !== null; + const conclusion = useReportedConclusion ? reportedConclusion : apiConclusion; + const completedAt = useReportedConclusion + ? observedAt + : status === "completed" + ? isoTimestamp(run.updated_at) + : null; + if ( + Number(run.id) !== identity.runId || + repository !== workflowRepository || + workflowPath !== `.github/workflows/${REPAIR_WORKFLOW}` || + !(status === "completed" || useReportedConclusion) || + !REPAIR_FAILURE_CONCLUSIONS.has(conclusion) || + !completedAt + ) { + return null; + } + return { + outcome: "repair_failed", + occurredAt: completedAt, + reason: `reconciled from ${REPAIR_WORKFLOW} conclusion ${conclusion}`, + runUrl: identity.url, + }; +} + +function terminalEvent(session: ActiveSession, terminal: Terminal) { return { event_type: "clawsweeper.automerge_metric", - event_id: `${session.session_id}:terminal:github-pr:${terminal.outcome}:${terminal.occurredAt}`, + // Reporters for the same outcome share an identity, while competing terminal + // observations stay distinct so event-level dedupe cannot choose the winner. + event_id: `${session.session_id}:terminal:reconcile:${terminal.outcome}`, session_id: session.session_id, phase: "terminal", occurred_at: terminal.occurredAt, @@ -196,13 +348,60 @@ function terminalEvent( policy_version: session.policy_version || "immediate-v1", state: null, outcome: terminal.outcome, - reason: `reconciled from authoritative GitHub PR ${terminal.outcome === "merged" ? "merged_at" : "closed_at"}`, + reason: terminal.reason, pr_url: session.pr_url || `https://github.com/${session.repository}/pull/${session.item_number}`, - run_url: session.run_url || null, + run_url: terminal.runUrl ?? session.run_url ?? null, }; } +function repairRunSuccessfullyRequeued(value: unknown) { + if (!value || typeof value !== "object") return null; + const payload = value as { total_count?: unknown; jobs?: unknown }; + if (!Array.isArray(payload.jobs)) return null; + const totalCount = Number(payload.total_count); + if (!Number.isInteger(totalCount) || totalCount < 0 || totalCount > payload.jobs.length) { + return null; + } + return payload.jobs.some((job) => { + if (!job || typeof job !== "object") return false; + const steps = (job as { steps?: unknown }).steps; + return ( + Array.isArray(steps) && + steps.some( + (step) => + step && + typeof step === "object" && + String((step as Record).name ?? "") === REPAIR_REQUEUE_DISPATCH_STEP && + String((step as Record).conclusion ?? "").toLowerCase() === "success", + ) + ); + }); +} + +function parseSessionId(value: string) { + const match = /^([A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+)#([1-9]\d*):([^:]+):(.+)$/.exec(value); + if (!match || !Number.isFinite(Date.parse(match[4]!))) return null; + return { repository: match[1]!, itemNumber: Number(match[2]!) }; +} + +function parseRunUrl(value: string) { + try { + const url = new URL(value); + const match = /^\/([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)\/actions\/runs\/([1-9]\d*)\/?$/.exec( + url.pathname, + ); + if (url.protocol !== "https:" || url.hostname !== "github.com" || !match) return null; + return { + repository: `${match[1]}/${match[2]}`, + runId: Number(match[3]), + url: `https://github.com/${match[1]}/${match[2]}/actions/runs/${match[3]}`, + }; + } catch { + return null; + } +} + async function fetchJson( url: URL, fetcher: typeof fetch, @@ -214,6 +413,51 @@ async function fetchJson( return response.json() as Promise; } +function githubApiUrl(repository: string, path: string) { + const [owner, name] = repository.split("/"); + return new URL( + `/repos/${encodeURIComponent(owner!)}/${encodeURIComponent(name!)}${path}`, + "https://api.github.com", + ); +} + +function githubHeaders(token: string) { + return { + Accept: "application/vnd.github+json", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "openclaw-clawsweeper-automerge-reconciler", + ...(token ? { Authorization: `Bearer ${token}` } : {}), + }; +} + +function sessionSkip(session: ActiveSession, githubReads: number, skipped: string) { + return { + session_id: session.session_id, + githubReads, + terminal: false, + delivered: false, + skipped, + error: null, + }; +} + +function skippedResult(skipped: string, candidates: number) { + return { + ok: true, + skipped, + candidates, + github_reads: 0, + terminal: 0, + delivered: 0, + failed: 0, + }; +} + +function repositoryName(value: unknown) { + const text = String(value ?? "").trim(); + return /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(text) ? text : null; +} + function isoTimestamp(value: unknown) { const timestamp = Date.parse(String(value ?? "")); return Number.isFinite(timestamp) ? new Date(timestamp).toISOString() : null; @@ -224,6 +468,11 @@ function boundedInt(value: unknown, fallback: number, maximum: number) { return Number.isInteger(parsed) && parsed > 0 ? Math.min(parsed, maximum) : fallback; } +function nullableText(value: unknown) { + const text = String(value ?? "").trim(); + return text || null; +} + function trimTrailingSlash(value: string) { return String(value).replace(/\/+$/, ""); } @@ -232,8 +481,63 @@ function errorMessage(error: unknown) { return error instanceof Error ? error.message : String(error); } +function usage() { + return `Usage: node scripts/dashboard-reconcile-automerge.ts [options] + +Reconcile active automerge sessions against authoritative PR and repair workflow state. + +Options: + --session-id Reconcile one active automerge session + --run-url Override that session's associated GitHub Actions run URL + --run-conclusion + Report a failure already established by the current workflow + --help Show this help + +Output is one JSON object. Exit codes: 0 completed/skipped safely, 1 operational +failure, 2 invalid CLI arguments.`; +} + +function parseCliArgs(argv: string[]) { + let sessionId: string | null = null; + let runUrl: string | null = null; + let runConclusion: string | null = null; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--help") return { help: true, sessionId, runUrl, runConclusion }; + if (arg !== "--session-id" && arg !== "--run-url" && arg !== "--run-conclusion") { + throw new Error(`unknown argument: ${arg}`); + } + const value = argv[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`${arg} requires a value`); + if (arg === "--session-id") sessionId = value; + else if (arg === "--run-url") runUrl = value; + else runConclusion = value; + index += 1; + } + if (runUrl && !sessionId) throw new Error("--run-url requires --session-id"); + if (runConclusion && (!sessionId || !runUrl)) { + throw new Error("--run-conclusion requires --session-id and --run-url"); + } + return { help: false, sessionId, runUrl, runConclusion }; +} + +export function automergeReconcileExitCode(result: { ok?: unknown; failed?: unknown }) { + return result.ok === true && Number(result.failed ?? 0) === 0 ? 0 : 1; +} + const invokedPath = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : ""; if (invokedPath === import.meta.url) { - const result = await reconcileAutomergeProductMetrics(); - console.log(JSON.stringify(result, null, 2)); + try { + const args = parseCliArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + } else { + const result = await reconcileAutomergeProductMetrics(args); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = automergeReconcileExitCode(result); + } + } catch (error) { + console.error(JSON.stringify({ ok: false, error: errorMessage(error) }, null, 2)); + process.exitCode = 2; + } } diff --git a/scripts/e2e/automerge.mjs b/scripts/e2e/automerge.mjs index a519d758cd..b9becde1dd 100755 --- a/scripts/e2e/automerge.mjs +++ b/scripts/e2e/automerge.mjs @@ -3,14 +3,28 @@ /** * Definition: run the repository-owned ClawSweeper automerge E2E harness. * Parameters: --scenario, --candidate-root, --output, and --keep are optional. - * Outputs: step logs plus summary.json under test-results/automerge by default; - * exit 0 means the selected production flow reached its asserted terminal state. + * Outputs: proof summaries under test-results/automerge by default. In + * candidate mode, a known product violation exits non-zero; in reproducer mode, + * the same exact fingerprint exits zero. + * Safety: every GitHub-looking token/API path in this harness is backed by a + * local fake gh binary and local bare remotes; live GitHub mutation is out of + * scope for this stable-red phase. * Decision: external commands fail closed so newly introduced GitHub dependencies * cannot silently turn this into a partial integration test. */ import path from "node:path"; -import { AUTOMERGE_E2E_SCENARIOS, runAutomergeE2E } from "../../test/e2e/automerge/run.mjs"; +import { runFlowScenario } from "../../test/e2e/automerge/flow-scenarios.mjs"; +import { writeGateInventory } from "../../test/e2e/automerge/gate-inventory.mjs"; +import { runModelScenario } from "../../test/e2e/automerge/model-scenarios.mjs"; +import { runPublicationScenario } from "../../test/e2e/automerge/publication-scenarios.mjs"; +import { AUTOMERGE_E2E_SCENARIOS } from "../../test/e2e/automerge/run.mjs"; +import { runRuntimeScenario } from "../../test/e2e/automerge/runtime-scenarios.mjs"; +import { + SCENARIO_IDS, + scenarioManifest, + scenariosForSuite, +} from "../../test/e2e/automerge/scenarios.mjs"; const args = parseArgs(process.argv.slice(2)); if (args.help) { @@ -23,22 +37,31 @@ Description: a real local Git remote. Options: - --scenario Scenario to run; use all for the full suite + --scenario Scenario to run; use all for the legacy flow suite (default: happy-path) + --suite flow, publication, model, or quick + --mode candidate or reproducer (default: candidate) + --repeat Repeat each selected scenario deterministically --expect success or setup-identity-failure (default: success) --list-scenarios Print supported scenario names --candidate-root Built ClawSweeper checkout to validate (default: cwd) + --openclaw-mirror Read-only OpenClaw checkout for real runtime fixtures --output Failure and proof artifact root --keep Keep the temporary scenario workspace -h, --help Show this help Outputs: //summary.json and one stdout/stderr log per production step. - Exit code 0 means all terminal-state and token-boundary assertions passed. + Candidate mode exits non-zero for a known product violation. Reproducer mode + exits zero only when the expected red fingerprint is reproduced exactly. Examples: pnpm e2e:automerge pnpm e2e:automerge -- --scenario all --output test-results/automerge + pnpm e2e:automerge -- --scenario publisher-depth1-concurrent-sibling \ + --candidate-root ../clawsweeper-p1 --mode reproducer + pnpm e2e:automerge -- --scenario openclaw-110725-git-hooks-path \ + --candidate-root ../clawsweeper-3beb5f0 --openclaw-mirror ../openclaw --mode reproducer pnpm e2e:automerge -- --scenario ci-regression-29623139111 \ --candidate-root ../clawsweeper-ci-regression --expect setup-identity-failure `); @@ -46,27 +69,84 @@ Examples: } if (args.listScenarios) { - process.stdout.write(`${AUTOMERGE_E2E_SCENARIOS.join("\n")}\n`); + process.stdout.write(`${SCENARIO_IDS.join("\n")}\n`); process.exit(0); } try { const selected = String(args.scenario ?? "happy-path"); - const scenarios = selected === "all" ? AUTOMERGE_E2E_SCENARIOS : [selected]; - const results = scenarios.map((scenario) => - runAutomergeE2E({ - candidateRoot: path.resolve(String(args.candidateRoot ?? process.cwd())), - outputRoot: path.resolve( - String(args.output ?? path.join(process.cwd(), "test-results", "automerge")), - ), - expectedOutcome: String(args.expect ?? "success"), - scenario, - keep: Boolean(args.keep), - }), + const mode = String(args.mode ?? "candidate"); + if (!["candidate", "reproducer"].includes(mode)) throw new Error(`unsupported mode: ${mode}`); + if (args.suite && args.scenario) throw new Error("--suite and --scenario are mutually exclusive"); + const repeat = positiveInteger(args.repeat ?? "1", "--repeat"); + const selectedManifests = args.suite + ? scenariosForSuite(String(args.suite)) + : selected === "all" + ? AUTOMERGE_E2E_SCENARIOS.map(scenarioManifest) + : [scenarioManifest(selected)]; + const manifests = + mode === "reproducer" && args.suite + ? selectedManifests.filter(({ expectedFingerprint }) => expectedFingerprint) + : selectedManifests; + if (manifests.length === 0) { + throw new Error("reproducer suites require at least one scenario with an expected fingerprint"); + } + if (args.expect !== undefined && manifests.some(({ kind }) => kind !== "flow")) { + throw new Error("--expect is supported only for legacy flow scenarios"); + } + const candidateRoot = path.resolve(String(args.candidateRoot ?? process.cwd())); + const outputRoot = path.resolve( + String(args.output ?? path.join(process.cwd(), "test-results", "automerge")), ); + const mirrorInput = args.openclawMirror ?? process.env.CLAWSWEEPER_E2E_OPENCLAW_MIRROR; + if (manifests.some(({ kind }) => kind === "runtime") && !mirrorInput) { + throw new Error("runtime scenarios require --openclaw-mirror"); + } + const results = []; + for (let attempt = 1; attempt <= repeat; attempt += 1) { + const attemptOutputRoot = + repeat === 1 + ? outputRoot + : path.join(outputRoot, "attempts", String(attempt).padStart(2, "0")); + for (const manifest of manifests) { + const common = { + candidateRoot, + outputRoot: attemptOutputRoot, + scenario: manifest.id, + mode, + }; + if (manifest.kind === "publication") { + results.push(runPublicationScenario({ ...common, keep: Boolean(args.keep) })); + } else if (manifest.kind === "runtime") { + results.push( + runRuntimeScenario({ + ...common, + openclawMirror: path.resolve(String(mirrorInput)), + keep: Boolean(args.keep), + }), + ); + } else if (manifest.kind === "model") { + results.push(runModelScenario(common)); + } else { + results.push( + runFlowScenario({ + ...common, + expectedOutcome: String(args.expect ?? manifest.legacyExpectedOutcome ?? "success"), + keep: Boolean(args.keep), + }), + ); + } + } + } + const { inventory, file: inventoryFile } = writeGateInventory({ mode, outputRoot, results }); process.stdout.write( - `${JSON.stringify(selected === "all" ? { status: "passed", results } : results[0], null, 2)}\n`, + `${JSON.stringify( + results.length > 1 ? { inventory, inventory_file: inventoryFile, results } : results[0], + null, + 2, + )}\n`, ); + process.exitCode = inventory.gate_exit_code; } catch (error) { process.stderr.write( `${error instanceof Error ? (error.stack ?? error.message) : String(error)}\n`, @@ -83,14 +163,26 @@ function parseArgs(argv) { else if (arg === "--list-scenarios") out.listScenarios = true; else if (arg === "--keep") out.keep = true; else if (arg === "--scenario") out.scenario = requiredValue(argv, ++index, arg); + else if (arg === "--suite") out.suite = requiredValue(argv, ++index, arg); + else if (arg === "--mode") out.mode = requiredValue(argv, ++index, arg); + else if (arg === "--repeat") out.repeat = requiredValue(argv, ++index, arg); else if (arg === "--expect") out.expect = requiredValue(argv, ++index, arg); else if (arg === "--candidate-root") out.candidateRoot = requiredValue(argv, ++index, arg); - else if (arg === "--output") out.output = requiredValue(argv, ++index, arg); + else if (arg === "--openclaw-mirror") { + out.openclawMirror = requiredValue(argv, ++index, arg); + } else if (arg === "--output") out.output = requiredValue(argv, ++index, arg); else throw new Error(`unknown option: ${arg}; use --help for usage`); } return out; } +function positiveInteger(value, option) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 1) + throw new Error(`${option} must be a positive integer`); + return parsed; +} + function requiredValue(argv, index, option) { const value = argv[index]; if (!value || value.startsWith("--")) throw new Error(`${option} requires a value`); diff --git a/test/automerge-metrics.test.ts b/test/automerge-metrics.test.ts index fad2506038..eb3f9ef1ad 100644 --- a/test/automerge-metrics.test.ts +++ b/test/automerge-metrics.test.ts @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; import test from "node:test"; import { mergeAutomergeMetricLedger, @@ -11,10 +12,40 @@ import { latestAutomergeActivationForCommand, postAutomergeMetricBestEffort, } from "../src/repair/automerge-product-telemetry.ts"; -import { reconcileAutomergeProductMetrics } from "../scripts/dashboard-reconcile-automerge.ts"; +import { + automergeReconcileExitCode, + reconcileAutomergeProductMetrics, +} from "../scripts/dashboard-reconcile-automerge.ts"; const now = "2026-07-17T12:00:00.000Z"; +test("reconcile CLI documents JSON output and explicit exit codes", () => { + const help = spawnSync(process.execPath, ["scripts/dashboard-reconcile-automerge.ts", "--help"], { + encoding: "utf8", + }); + assert.equal(help.status, 0); + assert.match(help.stdout, /--session-id /); + assert.match(help.stdout, /Exit codes: 0 completed\/skipped safely, 1 operational/); + + const invalid = spawnSync( + process.execPath, + [ + "scripts/dashboard-reconcile-automerge.ts", + "--run-url", + "https://github.com/openclaw/clawsweeper/actions/runs/1", + ], + { encoding: "utf8" }, + ); + assert.equal(invalid.status, 2); + assert.deepEqual(JSON.parse(invalid.stderr), { + ok: false, + error: "--run-url requires --session-id", + }); + assert.equal(automergeReconcileExitCode({ ok: true, failed: 0 }), 0); + assert.equal(automergeReconcileExitCode({ ok: true, failed: 1 }), 1); + assert.equal(automergeReconcileExitCode({ ok: false, failed: 0 }), 1); +}); + function event( session: string, phase: AutomergeMetricEvent["phase"], @@ -356,7 +387,7 @@ test("reconciliation caps active-session and GitHub reads", async () => { assert.equal(result.candidates, 3); assert.equal(result.github_reads, 3); assert.equal(githubReads.length, 3); - assert.match(metricsUrl, /range=24h/); + assert.match(metricsUrl, /range=7d/); assert.match(metricsUrl, /active_only=true/); assert.match(metricsUrl, /session_limit=3/); }); @@ -377,3 +408,342 @@ test("active-only metric sessions are bounded and oldest first", () => { ["active-1", "active-2"], ); }); + +for (const conclusion of ["failure", "timed_out", "action_required"]) { + test(`open PR with ${conclusion} repair run becomes repair_failed`, async () => { + const posted = []; + const sessionId = "openclaw/openclaw#42:100:2026-07-17T10:00:00Z"; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + sessionId, + fetcher: reconcileFetcher({ + session: activeSession(sessionId), + pr: { state: "open", merged_at: null, closed_at: null }, + run: repairRun(conclusion), + posted, + }), + }); + + assert.equal(result.terminal, 1); + assert.equal(result.delivered, 1); + assert.equal(result.github_reads, 3); + assert.equal(posted[0]?.outcome, "repair_failed"); + assert.equal(posted[0]?.event_id, `${sessionId}:terminal:reconcile:repair_failed`); + assert.equal(posted[0]?.occurred_at, "2026-07-17T11:30:00.000Z"); + }); +} + +test("failed workflow reports immediately while its GitHub run is still finalizing", async () => { + const posted = []; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + sessionId: "openclaw/openclaw#42:100:2026-07-17T10:00:00Z", + runUrl: "https://github.com/openclaw/clawsweeper/actions/runs/9001", + runConclusion: "failure", + fetcher: reconcileFetcher({ + session: activeSession(), + pr: { state: "open", merged_at: null, closed_at: null }, + run: repairRun(null, "in_progress"), + posted, + }), + }); + + assert.equal(result.delivered, 1); + assert.equal(posted[0]?.outcome, "repair_failed"); + assert.equal(posted[0]?.occurred_at, now); +}); + +test("completed GitHub run conclusion overrides an early workflow failure report", async () => { + const posted = []; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + sessionId: "openclaw/openclaw#42:100:2026-07-17T10:00:00Z", + runUrl: "https://github.com/openclaw/clawsweeper/actions/runs/9001", + runConclusion: "failure", + fetcher: reconcileFetcher({ + session: activeSession(), + pr: { state: "open", merged_at: null, closed_at: null }, + run: repairRun("success", "completed"), + posted, + }), + }); + + assert.equal(result.terminal, 0); + assert.equal(posted.length, 0); +}); + +test("scheduled reconciliation preserves sessions whose failed run requeued successfully", async () => { + const posted = []; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + fetcher: reconcileFetcher({ + session: activeSession(), + pr: { state: "open", merged_at: null, closed_at: null }, + run: repairRun("failure"), + jobs: repairJobs("success"), + posted, + }), + }); + + assert.equal(result.terminal, 0); + assert.equal(result.github_reads, 3); + assert.equal(result.results[0]?.skipped, "repair_run_successfully_requeued"); + assert.equal(posted.length, 0); +}); + +test("competing terminal outcomes keep distinct event identities", async () => { + const posted = []; + const session = activeSession(); + for (const pr of [ + { state: "open", merged_at: null, closed_at: null }, + { state: "closed", merged_at: "2026-07-17T11:45:00Z", closed_at: "2026-07-17T11:45:00Z" }, + ]) { + await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + fetcher: reconcileFetcher({ + session, + pr, + run: repairRun("failure"), + posted, + }), + }); + } + + assert.deepEqual( + posted.map((row) => row.event_id), + [ + `${session.session_id}:terminal:reconcile:repair_failed`, + `${session.session_id}:terminal:reconcile:merged`, + ], + ); +}); + +for (const [status, conclusion] of [ + ["in_progress", null], + ["completed", "success"], + ["completed", "cancelled"], + ["completed", "neutral"], + ["completed", "skipped"], +]) { + test(`repair run ${status}/${conclusion ?? "none"} stays active`, async () => { + const posted = []; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + fetcher: reconcileFetcher({ + session: activeSession(), + pr: { state: "open", merged_at: null, closed_at: null }, + run: repairRun(conclusion, status), + posted, + }), + }); + + assert.equal(result.terminal, 0); + assert.equal(result.delivered, 0); + assert.equal(posted.length, 0); + }); +} + +test("merged and closed PR state takes priority without reading a failed run", async () => { + for (const [pr, outcome] of [ + [ + { + state: "closed", + merged_at: "2026-07-17T11:20:00Z", + closed_at: "2026-07-17T11:21:00Z", + }, + "merged", + ], + [{ state: "closed", merged_at: null, closed_at: "2026-07-17T11:21:00Z" }, "pr_closed"], + ]) { + const posted = []; + let runReads = 0; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + fetcher: reconcileFetcher({ + session: activeSession(), + pr, + run: repairRun("failure"), + posted, + onRunRead: () => (runReads += 1), + }), + }); + + assert.equal(result.github_reads, 1); + assert.equal(runReads, 0); + assert.equal(posted[0]?.outcome, outcome); + } +}); + +test("invalid and external repair run URLs are conservative skips", async () => { + for (const runUrl of ["not-a-url", "https://github.com/example/foreign/actions/runs/9001"]) { + const posted = []; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + fetcher: reconcileFetcher({ + session: activeSession(undefined, runUrl), + pr: { state: "open", merged_at: null, closed_at: null }, + run: repairRun("failure"), + posted, + }), + }); + + assert.equal(result.terminal, 0); + assert.equal(result.github_reads, 1); + assert.equal(posted.length, 0); + } +}); + +test("repair workflow repository and identity must match", async () => { + for (const run of [ + repairRun("failure", "completed", "example/foreign"), + { ...repairRun("failure"), path: ".github/workflows/other.yml@refs/heads/main" }, + ]) { + const posted = []; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + fetcher: reconcileFetcher({ + session: activeSession(), + pr: { state: "open", merged_at: null, closed_at: null }, + run, + posted, + }), + }); + assert.equal(result.terminal, 0); + assert.equal(posted.length, 0); + } +}); + +test("seven-day lookback includes legacy failed sessions and exact-session filtering", async () => { + const sessionId = "openclaw/openclaw#42:100:2026-07-11T10:00:00Z"; + let metricsUrl = ""; + const posted = []; + const result = await reconcileAutomergeProductMetrics({ + env: reconcileEnv(), + now, + sessionId, + fetcher: reconcileFetcher({ + session: activeSession(sessionId, undefined, "2026-07-11T12:00:00Z"), + pr: { state: "open", merged_at: null, closed_at: null }, + run: repairRun("failure"), + posted, + onMetricsRead: (url) => (metricsUrl = url), + }), + }); + + assert.equal(result.delivered, 1); + assert.match(metricsUrl, /range=7d/); + assert.match(metricsUrl, /session_id=openclaw%2Fopenclaw%2342/); + assert.match(metricsUrl, /session_limit=1/); +}); + +test("merged plus repair_failed produces a 50 percent terminal success rate", () => { + const data = summarizeAutomergeMetrics( + ledger([ + event("merged-session", "activated", "2026-07-17T08:00:00Z"), + event("merged-session", "terminal", "2026-07-17T09:00:00Z", { + outcome: "merged", + }), + event("failed-session", "activated", "2026-07-17T09:00:00Z"), + event("failed-session", "terminal", "2026-07-17T10:00:00Z", { + outcome: "repair_failed", + }), + ]), + { range: "6h", now }, + ); + + assert.equal(data.summary.terminal_sessions, 2); + assert.equal(data.summary.merge_success_rate_percent, 50); + assert.equal(data.terminal_outcomes.repair_failed, 1); + assert.equal( + data.sessions.find((row) => row.session_id === "failed-session")?.state, + "repair_failed", + ); +}); + +function reconcileEnv() { + return { + CLAWSWEEPER_STATUS_URL: "https://status.example.test", + CLAWSWEEPER_STATUS_INGEST_TOKEN: "token", + }; +} + +function activeSession( + sessionId = "openclaw/openclaw#42:100:2026-07-17T10:00:00Z", + runUrl = "https://github.com/openclaw/clawsweeper/actions/runs/9001", + lastEventAt = "2026-07-17T10:00:00Z", +) { + return { + session_id: sessionId, + repository: "openclaw/openclaw", + item_number: 42, + policy_version: "immediate-v1", + last_event_at: lastEventAt, + terminal_at: null, + run_url: runUrl, + }; +} + +function repairRun(conclusion, status = "completed", repository = "openclaw/clawsweeper") { + return { + id: 9001, + status, + conclusion, + updated_at: "2026-07-17T11:30:00Z", + path: ".github/workflows/repair-cluster-worker.yml@refs/heads/main", + repository: { full_name: repository }, + }; +} + +function repairJobs(requeueConclusion = "skipped") { + return { + total_count: 1, + jobs: [ + { + steps: [ + { + name: "Requeue source-head repair races", + conclusion: requeueConclusion, + }, + ], + }, + ], + }; +} + +function reconcileFetcher({ + session, + pr, + run, + jobs = repairJobs(), + posted, + onRunRead, + onMetricsRead, +}) { + return async (input, init) => { + const url = String(input); + if (url.includes("/api/automerge-metrics")) { + onMetricsRead?.(url); + return Response.json({ sessions: [session] }); + } + if (url.includes("/pulls/42")) return Response.json(pr); + if (url.includes("/actions/runs/9001")) { + onRunRead?.(); + if (url.includes("/jobs?")) return Response.json(jobs); + return Response.json(run); + } + if (url.endsWith("/api/events")) { + posted.push(JSON.parse(String(init?.body))); + return new Response("", { status: 200 }); + } + throw new Error(`unexpected request: ${url}`); + }; +} diff --git a/test/dashboard-worker.test.ts b/test/dashboard-worker.test.ts index 71f3652dab..91999f1cf9 100644 --- a/test/dashboard-worker.test.ts +++ b/test/dashboard-worker.test.ts @@ -6660,6 +6660,8 @@ test("dashboard HTML preserves UTF-8 emoji labels", async () => { assert.match(html, /Time-window coverage/); assert.match(html, /Active sessions/); assert.match(html, /No terminal samples yet/); + assert.match(html, /Repair workflow failed/); + assert.match(html, /outcomeLabels\[session\.state\]/); assert.match(html, /Showing up to 30 latest sessions in the selected window/); assert.match(html, /Closed by ClawSweeper/); assert.match(html, /Worker Health/); diff --git a/test/e2e/automerge/candidate-revision.mjs b/test/e2e/automerge/candidate-revision.mjs new file mode 100644 index 0000000000..baf1b77a9d --- /dev/null +++ b/test/e2e/automerge/candidate-revision.mjs @@ -0,0 +1,167 @@ +import { execFileSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +export function candidateRevision(candidateRoot) { + return execFileSync("/usr/bin/git", ["-C", path.resolve(candidateRoot), "rev-parse", "HEAD"], { + encoding: "utf8", + env: gitProofEnvironment(), + stdio: ["ignore", "pipe", "pipe"], + }).trim(); +} + +export function assertCandidateRevision(candidateRoot, expectedRevision) { + const actual = candidateRevision(candidateRoot); + if (expectedRevision && actual !== expectedRevision) { + throw new Error( + `ClawSweeper candidate revision mismatch: expected ${expectedRevision}, got ${actual}`, + ); + } + return actual; +} + +export function candidateIdentity( + candidateRoot, + expectedRevision, + expectedExecutableDigest, + expectedDependencyDigest, +) { + const clawsweeperSha = assertCandidateRevision(candidateRoot, expectedRevision); + const candidateExecutableDigest = candidateExecutableDigestForRoot(candidateRoot); + if (expectedExecutableDigest && candidateExecutableDigest !== expectedExecutableDigest) { + throw new Error( + `ClawSweeper candidate executable digest mismatch: expected ${expectedExecutableDigest}, got ${candidateExecutableDigest}`, + ); + } + const candidateDependencyDigest = candidateDependencyDigestForRoot(candidateRoot); + if (expectedDependencyDigest && candidateDependencyDigest !== expectedDependencyDigest) { + throw new Error( + `ClawSweeper candidate dependency digest mismatch: expected ${expectedDependencyDigest}, got ${candidateDependencyDigest}`, + ); + } + return { + clawsweeperSha, + candidateDependencyDigest, + candidateExecutableDigest, + }; +} + +export function assertCandidateIdentityUnchanged(candidateRoot, before) { + const after = candidateIdentity(candidateRoot); + if (after.clawsweeperSha !== before.clawsweeperSha) { + throw new Error( + `ClawSweeper candidate revision changed during scenario: expected ${before.clawsweeperSha}, got ${after.clawsweeperSha}`, + ); + } + if (after.candidateExecutableDigest !== before.candidateExecutableDigest) { + throw new Error( + `ClawSweeper candidate executable digest changed during scenario: expected ${before.candidateExecutableDigest}, got ${after.candidateExecutableDigest}`, + ); + } + if (after.candidateDependencyDigest !== before.candidateDependencyDigest) { + throw new Error( + `ClawSweeper candidate dependency digest changed during scenario: expected ${before.candidateDependencyDigest}, got ${after.candidateDependencyDigest}`, + ); + } + return after; +} + +export function candidateExecutableDigest(candidateRoot) { + return candidateExecutableDigestForRoot(candidateRoot); +} + +export function candidateDependencyDigest(candidateRoot) { + return candidateDependencyDigestForRoot(candidateRoot); +} + +function candidateExecutableDigestForRoot(candidateRoot) { + const dist = path.join(path.resolve(candidateRoot), "dist"); + if (!fs.existsSync(dist)) throw new Error(`candidate executable directory is missing: ${dist}`); + const files = listFiles(dist); + if (files.length === 0) throw new Error(`candidate executable directory is empty: ${dist}`); + const hash = crypto.createHash("sha256"); + for (const file of files) { + const relative = path.relative(dist, file).split(path.sep).join("/"); + const stat = fs.statSync(file); + hash.update(relative).update("\0"); + hash.update(String(stat.mode & 0o777)).update("\0"); + hash.update(fs.readFileSync(file)).update("\0"); + } + return `sha256:${hash.digest("hex")}`; +} + +function listFiles(root) { + return fs + .readdirSync(root, { withFileTypes: true }) + .flatMap((entry) => { + const resolved = path.join(root, entry.name); + if (entry.isDirectory()) return listFiles(resolved); + if (entry.isFile()) return [resolved]; + throw new Error(`candidate executable tree contains unsupported entry: ${resolved}`); + }) + .sort(); +} + +function candidateDependencyDigestForRoot(candidateRoot) { + const root = path.resolve(candidateRoot); + const modules = path.join(root, "node_modules"); + if (!fs.existsSync(modules)) { + throw new Error(`candidate dependency directory is missing: ${modules}`); + } + const hash = crypto.createHash("sha256"); + for (const file of ["package.json", "pnpm-lock.yaml", "package-lock.json", "yarn.lock"]) { + const resolved = path.join(root, file); + if (!fs.existsSync(resolved)) continue; + hash.update(file).update("\0").update(fs.readFileSync(resolved)).update("\0"); + } + hash + .update("node_modules") + .update("\0") + .update(treeDigest(fs.realpathSync(modules), "candidate dependency tree")) + .update("\0"); + return `sha256:${hash.digest("hex")}`; +} + +function gitProofEnvironment() { + return { + GIT_CONFIG_COUNT: "0", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + ...(process.env.PATH === undefined ? {} : { PATH: process.env.PATH }), + }; +} + +function treeDigest(root, label) { + const files = listDependencyEntries(root); + if (files.length === 0) throw new Error(`${label} is empty: ${root}`); + const hash = crypto.createHash("sha256"); + for (const file of files) { + const relative = path.relative(root, file).split(path.sep).join("/"); + const stat = fs.lstatSync(file); + hash.update(relative).update("\0"); + hash.update(String(stat.mode & 0o777)).update("\0"); + if (stat.isSymbolicLink()) { + hash.update("symlink").update("\0").update(fs.readlinkSync(file)).update("\0"); + } else if (stat.isFile()) { + hash.update("file").update("\0").update(fs.readFileSync(file)).update("\0"); + } else if (stat.isDirectory()) { + hash.update("directory").update("\0"); + } else { + throw new Error(`${label} contains unsupported entry: ${file}`); + } + } + return `sha256:${hash.digest("hex")}`; +} + +function listDependencyEntries(root) { + return fs + .readdirSync(root, { withFileTypes: true }) + .flatMap((entry) => { + const resolved = path.join(root, entry.name); + if (entry.isDirectory()) return [resolved, ...listDependencyEntries(resolved)]; + return [resolved]; + }) + .sort(); +} diff --git a/test/e2e/automerge/candidate-revision.test.mjs b/test/e2e/automerge/candidate-revision.test.mjs new file mode 100644 index 0000000000..ceb20573ad --- /dev/null +++ b/test/e2e/automerge/candidate-revision.test.mjs @@ -0,0 +1,66 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + assertCandidateIdentityUnchanged, + candidateDependencyDigest, + assertCandidateRevision, + candidateExecutableDigest, + candidateIdentity, + candidateRevision, +} from "./candidate-revision.mjs"; + +test("candidate revision binding accepts only the exact checkout", () => { + const actual = candidateRevision(process.cwd()); + assert.equal(assertCandidateRevision(process.cwd(), actual), actual); + assert.throws( + () => assertCandidateRevision(process.cwd(), "0".repeat(40)), + /candidate revision mismatch/, + ); +}); + +test("candidate identity binds the executable dist contents", () => { + const actual = candidateRevision(process.cwd()); + const identity = candidateIdentity(process.cwd(), actual); + assert.equal(identity.clawsweeperSha, actual); + assert.equal(identity.candidateDependencyDigest, candidateDependencyDigest(process.cwd())); + assert.match(identity.candidateDependencyDigest, /^sha256:[0-9a-f]{64}$/); + assert.equal(identity.candidateExecutableDigest, candidateExecutableDigest(process.cwd())); + assert.match(identity.candidateExecutableDigest, /^sha256:[0-9a-f]{64}$/); + assert.throws( + () => candidateIdentity(process.cwd(), actual, `sha256:${"0".repeat(64)}`), + /candidate executable digest mismatch/, + ); + assert.throws( + () => + candidateIdentity( + process.cwd(), + actual, + identity.candidateExecutableDigest, + `sha256:${"0".repeat(64)}`, + ), + /candidate dependency digest mismatch/, + ); +}); + +test("candidate identity revalidation fails closed when executable bytes change", () => { + const actual = candidateRevision(process.cwd()); + const identity = candidateIdentity(process.cwd(), actual); + assert.deepEqual(assertCandidateIdentityUnchanged(process.cwd(), identity), identity); + assert.throws( + () => + assertCandidateIdentityUnchanged(process.cwd(), { + ...identity, + candidateExecutableDigest: `sha256:${"0".repeat(64)}`, + }), + /candidate executable digest changed during scenario/, + ); + assert.throws( + () => + assertCandidateIdentityUnchanged(process.cwd(), { + ...identity, + candidateDependencyDigest: `sha256:${"0".repeat(64)}`, + }), + /candidate dependency digest changed during scenario/, + ); +}); diff --git a/test/e2e/automerge/fake-gh.mjs b/test/e2e/automerge/fake-gh.mjs index 8cc3482143..d16252cd84 100755 --- a/test/e2e/automerge/fake-gh.mjs +++ b/test/e2e/automerge/fake-gh.mjs @@ -8,7 +8,18 @@ import path from "node:path"; const statePath = process.env.CLAWSWEEPER_E2E_GITHUB_STATE; if (!statePath) fail("CLAWSWEEPER_E2E_GITHUB_STATE is required"); const args = process.argv.slice(2); +const releaseStateLock = acquireStateLock(statePath); +process.on("exit", releaseStateLock); +for (const signal of ["SIGHUP", "SIGINT", "SIGTERM"]) { + process.on(signal, () => { + releaseStateLock(); + process.exit(128 + signalNumber(signal)); + }); +} const state = loadState(); +// Decision: these are synthetic capability labels consumed only by this fake gh +// process. Keeping the GH_TOKEN/GITHUB_TOKEN surface lets production code run +// unchanged while still failing closed before any real GitHub CLI can be used. const token = process.env.GH_TOKEN ?? process.env.GITHUB_TOKEN ?? ""; assertKnownToken(token); @@ -459,7 +470,118 @@ function loadState() { } function saveState() { - fs.writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`); + // The process-wide lock serializes local fake-gh read/modify/write cycles. + // Atomic replace still prevents readers from observing torn JSON if a future + // code path narrows the lock around only mutation operations. + const temporary = path.join( + path.dirname(statePath), + `${path.basename(statePath)}.${process.pid}.${Date.now()}.tmp`, + ); + fs.writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`); + fs.renameSync(temporary, statePath); +} + +function acquireStateLock(targetPath) { + const lock = `${targetPath}.lock`; + const owner = lockOwner(); + const deadline = Date.now() + 10_000; + while (true) { + const temporary = `${lock}.${process.pid}.${Date.now()}.tmp`; + try { + fs.writeFileSync(temporary, `${JSON.stringify(owner)}\n`, { mode: 0o600 }); + fs.linkSync(temporary, lock); + fs.unlinkSync(temporary); + return once(() => releaseLock(lock, owner)); + } catch (error) { + fs.rmSync(temporary, { force: true }); + if (error?.code !== "EEXIST") throw error; + reclaimStaleLock(lock); + if (Date.now() > deadline) fail(`timed out waiting for local fake-gh state lock: ${lock}`); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 10); + } + } +} + +function reclaimStaleLock(lock) { + const reclaimLock = `${lock}.reclaim`; + try { + fs.mkdirSync(reclaimLock); + } catch (error) { + if (error?.code === "EEXIST") return; + throw error; + } + try { + const owner = readLockOwner(lock); + if (owner && processIdentityExists(owner)) return; + if (!owner && !malformedLockIsOld(lock)) return; + try { + fs.unlinkSync(lock); + } catch { + // Another local fake-gh process won the recovery race; retry acquisition. + } + } finally { + fs.rmSync(reclaimLock, { recursive: true, force: true }); + } +} + +function releaseLock(lock, owner) { + const current = readLockOwner(lock); + if (JSON.stringify(current) !== JSON.stringify(owner)) return; + try { + fs.unlinkSync(lock); + } catch { + // Release is idempotent; the lock may already be gone during signal cleanup. + } +} + +function once(fn) { + let called = false; + return () => { + if (called) return; + called = true; + fn(); + }; +} + +function lockOwner() { + return { pid: process.pid, starttime: processStarttime(process.pid) }; +} + +function readLockOwner(lock) { + try { + return JSON.parse(fs.readFileSync(lock, "utf8")); + } catch { + return null; + } +} + +function malformedLockIsOld(lock) { + try { + return Date.now() - fs.statSync(lock).mtimeMs > 1_000; + } catch { + return false; + } +} + +function processIdentityExists(owner) { + if (!owner?.pid || !owner?.starttime) return true; + return processStarttime(owner.pid) === owner.starttime; +} + +function processStarttime(pid) { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + return stat.slice(stat.lastIndexOf(")") + 2).split(" ")[19] ?? null; + } catch { + return null; + } +} + +function signalNumber(signal) { + if (signal === "SIGHUP") return 1; + if (signal === "SIGINT") return 2; + if (signal === "SIGTERM") return 15; + return 1; } function respondJson(value, { paged = false } = {}) { diff --git a/test/e2e/automerge/flow-scenarios.mjs b/test/e2e/automerge/flow-scenarios.mjs new file mode 100644 index 0000000000..13fbbaf8a8 --- /dev/null +++ b/test/e2e/automerge/flow-scenarios.mjs @@ -0,0 +1,105 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { assertCandidateIdentityUnchanged, candidateIdentity } from "./candidate-revision.mjs"; +import { digestStrings, writeProofSummary } from "./proof-bundle.mjs"; +import { runAutomergeE2E } from "./run.mjs"; +import { scenarioManifest } from "./scenarios.mjs"; + +export function runFlowScenario({ + candidateRoot, + outputRoot, + scenario, + expectedOutcome = "success", + mode = "candidate", + keep = false, +}) { + const manifest = scenarioManifest(scenario); + if (manifest.kind !== "flow") throw new Error(`${scenario} is not a flow scenario`); + const startedAtMs = Date.now(); + const artifacts = path.resolve(outputRoot, scenario); + try { + const candidate = candidateIdentity(candidateRoot); + const result = runAutomergeE2E({ + candidateRoot, + outputRoot, + scenario, + expectedOutcome, + keep, + }); + assertCandidateIdentityUnchanged(candidateRoot, candidate); + if (result.status !== "passed") { + throw new Error(`legacy flow did not reach its asserted terminal status: ${result.status}`); + } + const legacySummary = JSON.parse(fs.readFileSync(path.join(artifacts, "summary.json"), "utf8")); + fs.writeFileSync( + path.join(artifacts, "flow-summary.json"), + `${JSON.stringify(legacySummary, null, 2)}\n`, + ); + const terminalState = flowTerminalState(legacySummary); + const productViolation = terminalState !== manifest.expectedProductOutcome; + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + productViolation, + clawsweeperSha: candidate.clawsweeperSha, + candidateDependencyDigest: candidate.candidateDependencyDigest, + candidateExecutableDigest: candidate.candidateExecutableDigest, + fixtureDigest: digestStrings([ + manifest.fixture, + scenario, + expectedOutcome, + JSON.stringify(legacySummary), + ]), + eventSequence: flowEventSequence(scenario), + phase: manifest.phase, + actualOutcome: terminalState, + terminalProductState: terminalState, + mergeCallCount: legacySummary.merge_commit ? 1 : 0, + invariant: flowInvariant(scenario), + }, + }); + return { ...result, summary }; + } catch (error) { + fs.mkdirSync(artifacts, { recursive: true }); + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + harnessError: true, + actualOutcome: "harness-error", + terminalProductState: "unknown", + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + }, + }); + return { status: "failed", scenario, artifacts, summary }; + } +} + +function flowTerminalState(legacySummary) { + return legacySummary.merge_commit ? "merged" : "blocked"; +} + +function flowEventSequence(scenario) { + return [ + "production-plan-review", + "artifact-handoff", + "production-execute-publish", + scenario === "pending-checks" ? "pending-check-observation" : "exact-head-observation", + "production-router-replay", + "product-terminal-state-oracle", + ]; +} + +function flowInvariant(scenario) { + if (scenario.includes("drift")) return "stale exact-head work must stop before merge mutation"; + if (scenario === "pending-checks") return "pending checks wait before exactly one eventual merge"; + if (scenario === "dependency-setup-mutation") + return "setup identity drift stops before Codex or push"; + return "the production flow converges with one logical terminal mutation"; +} diff --git a/test/e2e/automerge/gate-inventory.mjs b/test/e2e/automerge/gate-inventory.mjs new file mode 100644 index 0000000000..547e4fe426 --- /dev/null +++ b/test/e2e/automerge/gate-inventory.mjs @@ -0,0 +1,81 @@ +import fs from "node:fs"; +import path from "node:path"; + +import { gateExitCode } from "./proof-bundle.mjs"; + +export function writeGateInventory({ mode, outputRoot, results }) { + const grouped = Map.groupBy(results, (result) => result.summary.scenario_id); + const scenarios = [...grouped.entries()].map(([scenarioId, attempts]) => { + const summaries = attempts.map(({ summary }) => summary); + const signatures = new Set(summaries.map(stabilitySignature)); + return { + scenario_id: scenarioId, + attempts: summaries.length, + stable: signatures.size === 1, + gate_status: signatures.size === 1 ? summaries[0].gate_status : "unstable", + phase: signatures.size === 1 ? summaries[0].phase : null, + failure_fingerprint: signatures.size === 1 ? summaries[0].failure_fingerprint : null, + clawsweeper_sha: [...new Set(summaries.map(({ clawsweeper_sha }) => clawsweeper_sha))], + candidate_dependency_digest: [ + ...new Set(summaries.map(({ candidate_dependency_digest }) => candidate_dependency_digest)), + ], + candidate_executable_digest: [ + ...new Set(summaries.map(({ candidate_executable_digest }) => candidate_executable_digest)), + ], + wall_time_ms: summaries.reduce((total, summary) => total + summary.wall_time_ms, 0), + }; + }); + const stable = scenarios.every((scenario) => scenario.stable); + const exitCode = stable && results.every(({ summary }) => gateExitCode(summary) === 0) ? 0 : 1; + const inventory = { + schema_version: 1, + gate_mode: mode, + gate_status: inventoryGateStatus({ exitCode, scenarios, stable }), + gate_exit_code: exitCode, + scenarios, + }; + fs.mkdirSync(outputRoot, { recursive: true }); + const file = path.join(outputRoot, "failure-inventory.json"); + fs.writeFileSync(file, `${JSON.stringify(inventory, null, 2)}\n`); + return { inventory, file }; +} + +function inventoryGateStatus({ exitCode, scenarios, stable }) { + if (!stable) return "unstable"; + if (scenarios.every(({ gate_status }) => gate_status === "evidence-only")) { + return "evidence-only"; + } + if (exitCode === 0) return "passed"; + if (scenarios.some(({ gate_status }) => gate_status === "harness-error")) return "harness-error"; + return "candidate-red"; +} + +function stabilitySignature(summary) { + return JSON.stringify({ + gate_status: summary.gate_status, + phase: summary.phase, + failure_fingerprint: summary.failure_fingerprint, + clawsweeper_sha: summary.clawsweeper_sha, + candidate_dependency_digest: summary.candidate_dependency_digest, + candidate_executable_digest: summary.candidate_executable_digest, + openclaw_base_sha: summary.openclaw_base_sha, + openclaw_head_sha: summary.openclaw_head_sha, + fixture_digest: summary.fixture_digest, + event_sequence: summary.event_sequence, + expected_outcome: summary.expected_outcome, + actual_outcome: summary.actual_outcome, + terminal_product_state: summary.terminal_product_state, + merge_call_count: summary.merge_call_count, + git_refs_and_tree_digest: summary.git_refs_and_tree_digest, + child_process_snapshot: summary.child_process_snapshot, + error: normalizeErrorIdentity(summary.error), + }); +} + +function normalizeErrorIdentity(error) { + return String(error ?? "") + .replaceAll(/\b[0-9a-f]{40}\b/g, "") + .replaceAll(/\/tmp\/[^\s:)]+/g, "") + .replaceAll(/clawsweeper-[A-Za-z0-9_.-]+/g, "clawsweeper-") + .trim(); +} diff --git a/test/e2e/automerge/gate-inventory.test.mjs b/test/e2e/automerge/gate-inventory.test.mjs new file mode 100644 index 0000000000..2c3c7f89dd --- /dev/null +++ b/test/e2e/automerge/gate-inventory.test.mjs @@ -0,0 +1,68 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { writeGateInventory } from "./gate-inventory.mjs"; + +test("gate inventory rejects phase or fingerprint instability", () => { + const outputRoot = fs.mkdtempSync(path.join(os.tmpdir(), "automerge-inventory-")); + const base = { + scenario_id: "known-red", + gate_status: "reproducer-confirmed", + failure_fingerprint: "state.sibling-lost", + clawsweeper_sha: "a".repeat(40), + candidate_executable_digest: `sha256:${"b".repeat(64)}`, + fixture_digest: `sha256:${"c".repeat(64)}`, + event_sequence: ["publish"], + expected_outcome: "preserved", + actual_outcome: "lost", + terminal_product_state: "blocked", + merge_call_count: 0, + git_refs_and_tree_digest: `sha256:${"d".repeat(64)}`, + child_process_snapshot: [], + wall_time_ms: 1, + }; + const { inventory } = writeGateInventory({ + mode: "reproducer", + outputRoot, + results: [ + { summary: { ...base, phase: "publish" } }, + { summary: { ...base, phase: "different-phase" } }, + ], + }); + assert.equal(inventory.gate_status, "unstable"); + assert.equal(inventory.gate_exit_code, 1); +}); + +test("gate inventory rejects oracle output instability", () => { + const outputRoot = fs.mkdtempSync(path.join(os.tmpdir(), "automerge-inventory-")); + const base = { + scenario_id: "known-red", + gate_status: "reproducer-confirmed", + phase: "publish", + failure_fingerprint: "state.sibling-lost", + clawsweeper_sha: "a".repeat(40), + candidate_executable_digest: `sha256:${"b".repeat(64)}`, + fixture_digest: `sha256:${"c".repeat(64)}`, + event_sequence: ["publish"], + expected_outcome: "preserved", + actual_outcome: "lost", + terminal_product_state: "blocked", + merge_call_count: 0, + git_refs_and_tree_digest: `sha256:${"d".repeat(64)}`, + child_process_snapshot: [], + wall_time_ms: 1, + }; + const { inventory } = writeGateInventory({ + mode: "reproducer", + outputRoot, + results: [ + { summary: base }, + { summary: { ...base, git_refs_and_tree_digest: `sha256:${"e".repeat(64)}` } }, + ], + }); + assert.equal(inventory.gate_status, "unstable"); + assert.equal(inventory.gate_exit_code, 1); +}); diff --git a/test/e2e/automerge/git-proxy.mjs b/test/e2e/automerge/git-proxy.mjs index c46e865a84..13f00d9997 100755 --- a/test/e2e/automerge/git-proxy.mjs +++ b/test/e2e/automerge/git-proxy.mjs @@ -4,6 +4,22 @@ import { spawnSync } from "node:child_process"; import fs from "node:fs"; const rawArgs = process.argv.slice(2); +const publicationBarrier = publicationBarrierConfig(); +if (publicationBarrier && isPublicationBarrierCommand(rawArgs)) { + const child = spawnSync("/usr/bin/git", rawArgs, { + cwd: process.cwd(), + env: process.env, + encoding: "utf8", + }); + process.stdout.write(child.stdout ?? ""); + process.stderr.write(child.stderr ?? ""); + if (child.error) fail(child.error.message); + if ((child.status ?? 1) === 0 && claimPublicationBarrier(publicationBarrier.fired)) { + triggerPublicationWriter(publicationBarrier); + } + process.exit(child.status ?? 1); +} + const needsNetworkRewrite = rawArgs.some((arg) => /^https:\/\/github\.com\/[^/]+\/[^/]+\.git$/.test(arg), ); @@ -28,3 +44,62 @@ function fail(message) { process.stderr.write(`${message}\n`); process.exit(1); } + +function publicationBarrierConfig() { + const { + CLAWSWEEPER_E2E_PUBLICATION_BARRIER_FIRED: fired, + CLAWSWEEPER_E2E_PUBLICATION_WRITER_MODE: mode, + CLAWSWEEPER_E2E_PUBLICATION_WRITER_REMOTE: remote, + CLAWSWEEPER_E2E_PUBLICATION_WRITER_SCRIPT: script, + CLAWSWEEPER_E2E_PUBLICATION_WRITER_STATUS: status, + CLAWSWEEPER_E2E_PUBLICATION_WRITER_WORKSPACE: workspace, + } = process.env; + if (!fired || !mode || !remote || !script || !status || !workspace) return null; + return { fired, mode, remote, script, status, workspace }; +} + +function isPublicationBarrierCommand(args) { + return args.length === 2 && args[0] === "rev-parse" && args[1] === "HEAD"; +} + +function claimPublicationBarrier(fired) { + let fd; + try { + fd = fs.openSync(fired, "wx"); + fs.writeFileSync(fd, `${process.pid}\n`); + return true; + } catch (error) { + if (error && typeof error === "object" && "code" in error && error.code === "EEXIST") { + return false; + } + throw error; + } finally { + if (fd !== undefined) fs.closeSync(fd); + } +} + +function triggerPublicationWriter(config) { + const env = { ...process.env }; + for (const key of Object.keys(env)) { + if (key.startsWith("CLAWSWEEPER_E2E_PUBLICATION_")) delete env[key]; + } + const child = spawnSync( + process.execPath, + [config.script, config.mode, config.remote, config.workspace], + { + encoding: "utf8", + env, + }, + ); + fs.writeFileSync( + config.status, + `${JSON.stringify({ + error: child.error?.message ?? null, + status: child.status, + stderr: child.stderr, + stdout: child.stdout, + })}\n`, + ); + process.stdout.write(child.stdout ?? ""); + process.stderr.write(child.stderr ?? ""); +} diff --git a/test/e2e/automerge/immutable-ledger-worker.mjs b/test/e2e/automerge/immutable-ledger-worker.mjs new file mode 100644 index 0000000000..f85c41678b --- /dev/null +++ b/test/e2e/automerge/immutable-ledger-worker.mjs @@ -0,0 +1,100 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const [candidateRoot, ledgerRoot] = process.argv.slice(2); +if (!candidateRoot || !ledgerRoot) { + throw new Error("usage: immutable-ledger-worker.mjs "); +} + +const ledger = await import(pathToFileURL(path.join(candidateRoot, "dist/action-ledger.js")).href); +const input = actionEventInput(ledger, "keep_open"); +const created = ledger.writeActionEvent(ledgerRoot, input, { + now: () => new Date("2026-07-20T00:00:01.000Z"), +}); +const originalBytes = fs.readFileSync(created.path); +let conflict = null; +let conflictDetected = false; +try { + ledger.writeActionEvent(ledgerRoot, actionEventInput(ledger, "close"), { + now: () => new Date("2026-07-20T00:00:02.000Z"), + }); +} catch (error) { + if (error instanceof ledger.ActionEventConflictError) { + conflict = error.message; + conflictDetected = true; + } else throw error; +} +const finalBytes = fs.readFileSync(created.path); +process.stdout.write( + `${JSON.stringify({ + conflict, + conflict_detected: conflictDetected, + original_text: originalBytes.toString("utf8"), + original_preserved: originalBytes.equals(finalBytes), + relative_path: created.relativePath, + status: created.status, + })}\n`, +); + +function actionEventInput(ledgerModule, reasonCode) { + const repository = "openclaw/openclaw"; + const sourceRevision = "incident-head"; + const operationId = ledgerModule.actionOperationId(repository, "review", { + number: 110725, + sourceRevision, + }); + return { + eventKey: ledgerModule.actionEventKey("review.completed", { + repository, + number: 110725, + sourceRevision, + }), + operationId, + attemptId: ledgerModule.actionAttemptId(operationId, { + workflow: "automerge-e2e", + runId: "1", + runAttempt: 1, + }), + parentEventId: null, + phaseSeq: 1, + idempotencyKeySha256: ledgerModule.actionIdempotencyKey({ + repository, + number: 110725, + sourceRevision, + action: "review", + }), + type: ledgerModule.ACTION_EVENT_TYPES.reviewCompleted, + producer: { + repository: "openclaw/clawsweeper", + sha: "e2e-candidate", + workflow: "automerge-e2e", + job: "immutable-path", + runId: "1", + runAttempt: 1, + component: "review", + }, + subject: { + repository, + kind: "pull_request", + number: 110725, + sourceRevision, + recordPath: "records/openclaw-openclaw/items/110725.md", + }, + action: { + name: "review", + status: "completed", + reasonCode, + retryable: false, + mutation: false, + }, + privacy: { + classification: "internal", + redactionVersion: "v1", + fieldsDropped: [], + }, + occurredAt: "2026-07-20T00:00:00.000Z", + }; +} diff --git a/test/e2e/automerge/model-scenarios.mjs b/test/e2e/automerge/model-scenarios.mjs new file mode 100644 index 0000000000..25cf303573 --- /dev/null +++ b/test/e2e/automerge/model-scenarios.mjs @@ -0,0 +1,52 @@ +import fs from "node:fs"; + +import { candidateIdentity } from "./candidate-revision.mjs"; +import { artifactDirectory, digestStrings, writeProofSummary } from "./proof-bundle.mjs"; +import { scenarioManifest } from "./scenarios.mjs"; +import { evaluateStateModel } from "./state-model.mjs"; + +export function runModelScenario({ candidateRoot, outputRoot, scenario, mode = "candidate" }) { + const manifest = scenarioManifest(scenario); + if (manifest.kind !== "model") throw new Error(`${scenario} is not a model scenario`); + const startedAtMs = Date.now(); + const artifacts = artifactDirectory(outputRoot, scenario); + try { + const state = evaluateStateModel(scenario); + fs.writeFileSync( + `${artifacts}/event-sequence.json`, + `${JSON.stringify(state.eventSequence, null, 2)}\n`, + ); + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + evidenceOnly: true, + productViolation: false, + ...candidateIdentity(candidateRoot), + fixtureDigest: digestStrings([scenario, ...state.eventSequence]), + eventSequence: state.eventSequence, + actualOutcome: state.actualOutcome, + terminalProductState: state.terminalProductState, + mergeCallCount: state.mergeCallCount, + invariant: state.invariant, + }, + }); + return { summary, artifacts }; + } catch (error) { + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + harnessError: true, + actualOutcome: "harness-error", + terminalProductState: "unknown", + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + }, + }); + return { summary, artifacts }; + } +} diff --git a/test/e2e/automerge/proof-bundle.mjs b/test/e2e/automerge/proof-bundle.mjs new file mode 100644 index 0000000000..d7f7e62fc2 --- /dev/null +++ b/test/e2e/automerge/proof-bundle.mjs @@ -0,0 +1,134 @@ +import crypto from "node:crypto"; +import fs from "node:fs"; +import path from "node:path"; + +const VALID_GATE_STATUSES = new Set([ + "candidate-red", + "evidence-only", + "reproducer-confirmed", + "passed", + "harness-error", +]); + +export function artifactDirectory(outputRoot, scenarioId) { + const directory = path.resolve(outputRoot, scenarioId); + fs.rmSync(directory, { recursive: true, force: true }); + fs.mkdirSync(directory, { recursive: true }); + return directory; +} + +export function writeProofSummary({ artifacts, manifest, mode, observed, startedAtMs }) { + if (!["candidate", "reproducer"].includes(mode)) + throw new Error(`unsupported gate mode: ${mode}`); + const productViolation = Boolean(observed.productViolation); + const fingerprintMatches = + !manifest.expectedFingerprint || observed.failureFingerprint === manifest.expectedFingerprint; + const gateStatus = gateStatusFor({ + mode, + productViolation, + fingerprintMatches, + expectsFingerprint: Boolean(manifest.expectedFingerprint), + evidenceOnly: observed.evidenceOnly, + harnessError: observed.harnessError, + }); + const summary = { + scenario_id: manifest.id, + scenario_version: manifest.version, + gate_mode: mode, + gate_status: gateStatus, + gate_exit_code: gateStatus === "passed" || gateStatus === "reproducer-confirmed" ? 0 : 1, + clawsweeper_sha: observed.clawsweeperSha ?? null, + candidate_dependency_digest: observed.candidateDependencyDigest ?? null, + candidate_executable_digest: observed.candidateExecutableDigest ?? null, + openclaw_base_sha: observed.openclawBaseSha ?? null, + openclaw_head_sha: observed.openclawHeadSha ?? null, + fixture_digest: observed.fixtureDigest ?? null, + event_sequence: observed.eventSequence ?? [], + fault_point: manifest.faultPoint, + random_seed: manifest.randomSeed, + phase: observed.phase ?? manifest.phase, + expected_outcome: manifest.expectedProductOutcome, + actual_outcome: observed.actualOutcome ?? null, + failure_fingerprint: observed.failureFingerprint ?? null, + terminal_product_state: observed.terminalProductState ?? null, + merge_call_count: observed.mergeCallCount ?? 0, + git_refs_and_tree_digest: observed.gitRefsAndTreeDigest ?? null, + child_process_snapshot: observed.childProcessSnapshot ?? [], + wall_time_ms: Date.now() - startedAtMs, + cpu_time_ms: observed.cpuTimeMs ?? null, + peak_rss_bytes: observed.peakRssBytes ?? null, + disk_delta_bytes: observed.diskDeltaBytes ?? null, + invariant: observed.invariant ?? null, + error: + observed.error ?? + (mode === "reproducer" && !manifest.expectedFingerprint && gateStatus !== "evidence-only" + ? "reproducer mode requires an expected failure fingerprint" + : null) ?? + (mode === "reproducer" && manifest.expectedFingerprint && !productViolation + ? `expected failure was not reproduced: ${manifest.expectedFingerprint}` + : null), + }; + validateProofSummary(summary); + fs.writeFileSync(path.join(artifacts, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`); + return summary; +} + +export function validateProofSummary(summary) { + if (!VALID_GATE_STATUSES.has(summary.gate_status)) { + throw new Error(`invalid proof gate_status: ${summary.gate_status}`); + } + for (const key of [ + "scenario_id", + "scenario_version", + "event_sequence", + "gate_exit_code", + "phase", + "expected_outcome", + "merge_call_count", + "child_process_snapshot", + "wall_time_ms", + ]) { + if (summary[key] === undefined) throw new Error(`proof summary is missing ${key}`); + } + if (summary.gate_status === "reproducer-confirmed" && !summary.failure_fingerprint) { + throw new Error("a confirmed reproducer requires an exact failure fingerprint"); + } + if ( + summary.gate_mode === "reproducer" && + summary.gate_status !== "harness-error" && + summary.gate_status !== "evidence-only" && + !summary.failure_fingerprint + ) { + throw new Error("reproducer mode requires an expected failure fingerprint"); + } + if (summary.gate_exit_code !== gateExitCode(summary)) { + throw new Error("proof gate_exit_code does not match gate_status"); + } +} + +export function digestStrings(values) { + const hash = crypto.createHash("sha256"); + for (const value of values) hash.update(String(value)).update("\0"); + return `sha256:${hash.digest("hex")}`; +} + +export function gateExitCode(summary) { + return summary.gate_status === "passed" || summary.gate_status === "reproducer-confirmed" ? 0 : 1; +} + +function gateStatusFor({ + mode, + productViolation, + fingerprintMatches, + expectsFingerprint, + evidenceOnly, + harnessError, +}) { + if (harnessError) return "harness-error"; + if (mode === "reproducer" && !expectsFingerprint) return "harness-error"; + if (evidenceOnly) return "evidence-only"; + if (!productViolation) + return mode === "reproducer" && expectsFingerprint ? "harness-error" : "passed"; + if (expectsFingerprint && !fingerprintMatches) return "harness-error"; + return mode === "reproducer" ? "reproducer-confirmed" : "candidate-red"; +} diff --git a/test/e2e/automerge/proof-bundle.test.mjs b/test/e2e/automerge/proof-bundle.test.mjs new file mode 100644 index 0000000000..11a3419fb8 --- /dev/null +++ b/test/e2e/automerge/proof-bundle.test.mjs @@ -0,0 +1,153 @@ +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; + +import { artifactDirectory, gateExitCode, writeProofSummary } from "./proof-bundle.mjs"; + +const manifest = { + id: "known-red", + version: 1, + phase: "publication", + expectedProductOutcome: "preserved", + expectedFingerprint: "state.sibling-lost", + faultPoint: "before-cas", + randomSeed: 0, +}; + +test("candidate gate remains red when a product oracle fails", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "red"); + const summary = writeProofSummary({ + artifacts, + manifest, + mode: "candidate", + startedAtMs: Date.now(), + observed: { + productViolation: true, + failureFingerprint: "state.sibling-lost", + actualOutcome: "lost", + }, + }); + assert.equal(summary.gate_status, "candidate-red"); + assert.equal(summary.gate_exit_code, 1); + assert.equal(gateExitCode(summary), 1); +}); + +test("reproducer mode confirms the same exact red fingerprint", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "red"); + const summary = writeProofSummary({ + artifacts, + manifest, + mode: "reproducer", + startedAtMs: Date.now(), + observed: { + productViolation: true, + failureFingerprint: "state.sibling-lost", + actualOutcome: "lost", + }, + }); + assert.equal(summary.gate_status, "reproducer-confirmed"); + assert.equal(summary.gate_exit_code, 0); + assert.equal(gateExitCode(summary), 0); +}); + +test("candidate mode passes when a fingerprinted violation is fixed", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "fixed"); + const summary = writeProofSummary({ + artifacts, + manifest, + mode: "candidate", + startedAtMs: Date.now(), + observed: { + productViolation: false, + actualOutcome: "preserved", + }, + }); + assert.equal(summary.gate_status, "passed"); + assert.equal(summary.gate_exit_code, 0); +}); + +test("candidate mode treats a mismatched known violation as harness error", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "red"); + const summary = writeProofSummary({ + artifacts, + manifest, + mode: "candidate", + startedAtMs: Date.now(), + observed: { + productViolation: true, + failureFingerprint: "state.other", + actualOutcome: "lost", + }, + }); + assert.equal(summary.gate_status, "harness-error"); + assert.equal(summary.gate_exit_code, 1); +}); + +test("a different failure is a harness error, not a confirmed reproducer", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "red"); + const summary = writeProofSummary({ + artifacts, + manifest, + mode: "reproducer", + startedAtMs: Date.now(), + observed: { + productViolation: true, + failureFingerprint: "state.timeout", + actualOutcome: "timeout", + }, + }); + assert.equal(summary.gate_status, "harness-error"); +}); + +test("an expected red that does not reproduce fails closed", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "red"); + const summary = writeProofSummary({ + artifacts, + manifest, + mode: "reproducer", + startedAtMs: Date.now(), + observed: { + productViolation: false, + actualOutcome: "unexpected-pass", + }, + }); + assert.equal(summary.gate_status, "harness-error"); + assert.equal(summary.gate_exit_code, 1); + assert.match(summary.error, /expected failure was not reproduced/); +}); + +test("reproducer mode fails closed without an expected fingerprint", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "green"); + const summary = writeProofSummary({ + artifacts, + manifest: { ...manifest, expectedFingerprint: null }, + mode: "reproducer", + startedAtMs: Date.now(), + observed: { + productViolation: false, + actualOutcome: "passed", + }, + }); + assert.equal(summary.gate_status, "harness-error"); + assert.equal(summary.gate_exit_code, 1); + assert.match(summary.error, /requires an expected failure fingerprint/); +}); + +test("model-only evidence does not masquerade as a candidate pass", () => { + const artifacts = artifactDirectory(fs.mkdtempSync(path.join(os.tmpdir(), "proof-")), "model"); + const summary = writeProofSummary({ + artifacts, + manifest: { ...manifest, kind: "model", expectedFingerprint: null }, + mode: "candidate", + startedAtMs: Date.now(), + observed: { + evidenceOnly: true, + productViolation: false, + actualOutcome: "recovered", + }, + }); + assert.equal(summary.gate_status, "evidence-only"); + assert.equal(summary.gate_exit_code, 1); +}); diff --git a/test/e2e/automerge/publication-scenarios.mjs b/test/e2e/automerge/publication-scenarios.mjs new file mode 100644 index 0000000000..9647b0b27d --- /dev/null +++ b/test/e2e/automerge/publication-scenarios.mjs @@ -0,0 +1,398 @@ +import { execFileSync, spawnSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { assertCandidateIdentityUnchanged, candidateIdentity } from "./candidate-revision.mjs"; +import { artifactDirectory, digestStrings, writeProofSummary } from "./proof-bundle.mjs"; +import { scenarioManifest } from "./scenarios.mjs"; + +const helperRoot = path.dirname(fileURLToPath(import.meta.url)); +let publicationEnvironmentRoot = null; + +// Publication scenarios intentionally use production publishing code against +// disposable local bare remotes. No GitHub refs, comments, labels, or PR state +// are touched while reproducing the state-tree loss cases. +export function runPublicationScenario({ + candidateRoot, + outputRoot, + scenario, + mode = "candidate", + keep = false, +}) { + const manifest = scenarioManifest(scenario); + if (manifest.kind !== "publication") throw new Error(`${scenario} is not a publication scenario`); + const startedAtMs = Date.now(); + const artifacts = artifactDirectory(outputRoot, scenario); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-publication-e2e-")); + publicationEnvironmentRoot = root; + try { + const candidate = candidateIdentity( + candidateRoot, + mode === "reproducer" ? manifest.candidateRevision : null, + mode === "reproducer" ? manifest.candidateExecutableDigest : null, + mode === "reproducer" ? manifest.candidateDependencyDigest : null, + ); + if (scenario === "publisher-canonical-path-conflict") { + return immutableConflictScenario({ + artifacts, + candidate, + candidateRoot, + manifest, + mode, + root, + startedAtMs, + }); + } + const fixture = createShallowPublicationFixture(root); + const writerMode = scenario === "publisher-two-parent-then-depth1" ? "merge" : "sibling"; + fs.writeFileSync( + path.join(fixture.work, "results/local-publication.json"), + '{"publisher":1}\n', + ); + const barrier = publicationBarrier(root, writerMode, fixture.remote); + const worker = spawnLocalOnlyNode( + [ + path.join(helperRoot, "publication-worker.mjs"), + path.resolve(candidateRoot), + fixture.work, + JSON.stringify(["results"]), + ], + { encoding: "utf8", env: sanitizedEnvironment(root, barrier), timeout: 120_000 }, + ); + writeChildLog(artifacts, "publisher-one", worker); + assertCandidateIdentityUnchanged(candidateRoot, candidate); + const writer = readWriterStatus(barrier.status); + writeSyntheticChildLog(artifacts, "publisher-two", writer); + + const expectedPath = + writerMode === "merge" ? "results/merge-right.json" : "results/concurrent-sibling.json"; + const remoteHead = remoteRef(fixture.remote, "main"); + const treePaths = git(["--git-dir", fixture.remote, "ls-tree", "-r", "--name-only", remoteHead]) + .trim() + .split("\n") + .filter(Boolean) + .sort(); + const siblingPreserved = treePaths.includes(expectedPath); + const localPreserved = treePaths.includes("results/local-publication.json"); + const siblingContentPreserved = + siblingPreserved && + remoteFile(fixture.remote, remoteHead, expectedPath) === expectedContent(writerMode); + const localContentPreserved = + localPreserved && + remoteFile(fixture.remote, remoteHead, "results/local-publication.json") === + '{"publisher":1}\n'; + const baseContentPreserved = + treePaths.includes("results/base.json") && + remoteFile(fixture.remote, remoteHead, "results/base.json") === '{"base":true}\n'; + const harnessError = writer.status !== 0 || timedOut(worker); + const productViolation = + worker.status !== 0 || + !siblingContentPreserved || + !localContentPreserved || + !baseContentPreserved; + const failureFingerprint = productViolation + ? worker.status !== 0 + ? "state-publication.publisher-did-not-converge" + : !siblingContentPreserved + ? writerMode === "merge" + ? "state-publication.merge-tree-entry-lost" + : "state-publication.concurrent-sibling-lost" + : !localContentPreserved + ? "state-publication.local-entry-lost" + : "state-publication.base-entry-lost" + : null; + const graph = git([ + "--git-dir", + fixture.remote, + "log", + "--graph", + "--format=%H %P %s", + "--all", + ]); + fs.writeFileSync(path.join(artifacts, "git-graph.txt"), graph); + fs.writeFileSync(path.join(artifacts, "git-tree.txt"), `${treePaths.join("\n")}\n`); + + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + harnessError, + productViolation, + failureFingerprint, + clawsweeperSha: candidate.clawsweeperSha, + candidateDependencyDigest: candidate.candidateDependencyDigest, + candidateExecutableDigest: candidate.candidateExecutableDigest, + fixtureDigest: digestStrings([fixture.initialHead, ...treePaths]), + eventSequence: [ + "publisher-one-depth1-clone", + `publisher-two-${writerMode}-push`, + "publisher-one-production-publish", + "remote-tree-oracle", + ], + actualOutcome: productViolation ? "remote-tree-incomplete" : "remote-tree-preserved", + terminalProductState: productViolation ? "blocked" : "published", + gitRefsAndTreeDigest: digestStrings([remoteHead, ...treePaths]), + childProcessSnapshot: [ + { name: "publisher-two", exit_code: writer.status }, + { name: "publisher-one", exit_code: worker.status }, + ], + invariant: `publisher one must preserve ${expectedPath}, its own publication, and the initial state tree`, + error: worker.status === 0 ? null : normalizeError(`${worker.stderr}\n${worker.stdout}`), + }, + }); + return { summary, artifacts }; + } catch (error) { + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + harnessError: true, + phase: manifest.phase, + actualOutcome: "harness-error", + terminalProductState: "unknown", + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + }, + }); + return { summary, artifacts }; + } finally { + publicationEnvironmentRoot = null; + if (!keep) fs.rmSync(root, { recursive: true, force: true }); + } +} + +function createShallowPublicationFixture(root) { + const remote = path.join(root, "state.git"); + const seed = path.join(root, "seed"); + const work = path.join(root, "publisher-one"); + git(["init", "--bare", remote]); + git(["clone", remote, seed]); + configureUser(seed, "Fixture Seeder"); + write(path.join(seed, "results/base.json"), '{"base":true}\n'); + git(["add", "."], seed); + git(["commit", "-m", "chore: seed state"], seed); + git(["push", "origin", "HEAD:main"], seed); + git(["--git-dir", remote, "symbolic-ref", "HEAD", "refs/heads/main"]); + git(["clone", "--depth", "1", `file://${remote}`, work]); + configureUser(work, "Publisher One"); + return { remote, seed, work, initialHead: remoteRef(remote, "main") }; +} + +function immutableConflictScenario({ + artifacts, + candidate, + candidateRoot, + manifest, + mode, + root, + startedAtMs, +}) { + const fixture = createShallowPublicationFixture(root); + const ledgerRoot = path.join(fixture.work, "ledger-root"); + fs.mkdirSync(ledgerRoot); + const immutable = spawnLocalOnlyNode( + [path.join(helperRoot, "immutable-ledger-worker.mjs"), path.resolve(candidateRoot), ledgerRoot], + { encoding: "utf8", env: sanitizedEnvironment(root), timeout: 120_000 }, + ); + writeChildLog(artifacts, "immutable-ledger", immutable); + const observation = immutable.status === 0 ? parseLastJsonLine(immutable.stdout) : {}; + const publisher = spawnLocalOnlyNode( + [ + path.join(helperRoot, "publication-worker.mjs"), + path.resolve(candidateRoot), + fixture.work, + JSON.stringify(["ledger-root"]), + ], + { encoding: "utf8", env: sanitizedEnvironment(root), timeout: 120_000 }, + ); + writeChildLog(artifacts, "immutable-publisher", publisher); + assertCandidateIdentityUnchanged(candidateRoot, candidate); + const remoteHead = remoteRef(fixture.remote, "main"); + const treePaths = git(["--git-dir", fixture.remote, "ls-tree", "-r", "--name-only", remoteHead]) + .trim() + .split("\n") + .filter(Boolean) + .sort(); + const immutablePath = observation.relative_path + ? path.posix.join("ledger-root", String(observation.relative_path).replaceAll(path.sep, "/")) + : null; + const immutablePathExists = Boolean(immutablePath && treePaths.includes(immutablePath)); + const conflictBlocked = + immutable.status === 0 && + publisher.status === 0 && + observation.original_preserved === true && + immutablePathExists && + remoteFile(fixture.remote, remoteHead, immutablePath) === observation.original_text && + observation.conflict_detected === true; + const productViolation = !conflictBlocked; + const harnessError = + immutable.status !== 0 || publisher.status !== 0 || timedOut(immutable) || timedOut(publisher); + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + harnessError, + productViolation, + failureFingerprint: productViolation + ? "state-publication.immutable-path-not-preserved" + : null, + clawsweeperSha: candidate.clawsweeperSha, + candidateDependencyDigest: candidate.candidateDependencyDigest, + candidateExecutableDigest: candidate.candidateExecutableDigest, + fixtureDigest: digestStrings([fixture.initialHead, immutablePath, ...treePaths]), + eventSequence: [ + "production-canonical-path-create", + "production-conflicting-replay", + "production-state-publish", + "remote-tree-oracle", + ], + actualOutcome: conflictBlocked ? "blocked" : "immutable-path-not-preserved", + terminalProductState: conflictBlocked ? "blocked" : "unsafe", + gitRefsAndTreeDigest: digestStrings([remoteHead, ...treePaths]), + childProcessSnapshot: [ + { name: "immutable-ledger", exit_code: immutable.status }, + { name: "immutable-publisher", exit_code: publisher.status }, + ], + invariant: "an immutable canonical path cannot be overwritten with different bytes", + error: + immutable.status === 0 && publisher.status === 0 + ? null + : normalizeError(`${immutable.stderr}\n${publisher.stderr}`), + }, + }); + return { summary, artifacts }; +} + +function spawnLocalOnlyNode(args, options) { + // PATH shims prove command routing, but the phase boundary also requires an + // outer egress-deny layer that candidate code cannot bypass with absolute tools. + return spawnSync( + "/usr/bin/unshare", + ["--user", "--map-root-user", "--net", process.execPath, ...args], + options, + ); +} + +function parseLastJsonLine(stdout) { + const line = String(stdout).trim().split("\n").at(-1); + if (!line) throw new Error("publication worker produced no observation"); + return JSON.parse(line); +} + +function writeChildLog(artifacts, name, child) { + fs.writeFileSync(path.join(artifacts, `${name}.stdout.log`), child.stdout ?? ""); + fs.writeFileSync(path.join(artifacts, `${name}.stderr.log`), child.stderr ?? ""); +} + +function publicationBarrier(root, writerMode, remote) { + return { + fired: path.join(root, "publication-barrier-fired"), + mode: writerMode, + remote, + script: path.join(helperRoot, "publication-writer.mjs"), + status: path.join(root, "publication-writer-status.json"), + workspace: path.join(root, "publisher-two"), + }; +} + +function readWriterStatus(file) { + if (!fs.existsSync(file)) + return { status: 1, stderr: "publication writer barrier did not fire\n", stdout: "" }; + return JSON.parse(fs.readFileSync(file, "utf8")); +} + +function writeSyntheticChildLog(artifacts, name, child) { + fs.writeFileSync(path.join(artifacts, `${name}.stdout.log`), child.stdout ?? ""); + fs.writeFileSync(path.join(artifacts, `${name}.stderr.log`), child.stderr ?? ""); +} + +function sanitizedEnvironment(isolationRoot = null, barrier = null) { + const root = isolationRoot ?? fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-e2e-env-")); + const home = path.join(root, "home"); + const config = path.join(root, "config"); + const fakeBin = path.join(root, "fake-bin"); + for (const directory of [home, config, fakeBin]) fs.mkdirSync(directory, { recursive: true }); + const fakeGh = path.join(fakeBin, "gh"); + if (!fs.existsSync(fakeGh)) { + fs.writeFileSync( + fakeGh, + "#!/usr/bin/env sh\necho 'real gh disabled in local E2E' >&2\nexit 127\n", + ); + fs.chmodSync(fakeGh, 0o755); + } + const fakeGit = path.join(fakeBin, "git"); + if (!fs.existsSync(fakeGit)) { + fs.symlinkSync(path.join(helperRoot, "git-proxy.mjs"), fakeGit); + } + const env = { + GIT_AUTHOR_DATE: "2026-07-20T00:00:00Z", + GIT_COMMITTER_DATE: "2026-07-20T00:00:00Z", + GIT_CONFIG_COUNT: "0", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + HOME: home, + PATH: `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`, + TMPDIR: root, + TZ: "UTC", + XDG_CONFIG_HOME: config, + }; + if (barrier) { + env.CLAWSWEEPER_E2E_PUBLICATION_BARRIER_FIRED = barrier.fired; + env.CLAWSWEEPER_E2E_PUBLICATION_WRITER_MODE = barrier.mode; + env.CLAWSWEEPER_E2E_PUBLICATION_WRITER_REMOTE = barrier.remote; + env.CLAWSWEEPER_E2E_PUBLICATION_WRITER_SCRIPT = barrier.script; + env.CLAWSWEEPER_E2E_PUBLICATION_WRITER_STATUS = barrier.status; + env.CLAWSWEEPER_E2E_PUBLICATION_WRITER_WORKSPACE = barrier.workspace; + } + return env; +} + +function configureUser(worktree, name) { + git(["config", "user.name", name], worktree); + git(["config", "user.email", "e2e@example.invalid"], worktree); +} + +function remoteRef(remote, branch) { + return git(["--git-dir", remote, "rev-parse", `refs/heads/${branch}`]).trim(); +} + +function remoteFile(remote, revision, file) { + return git(["--git-dir", remote, "show", `${revision}:${file}`]); +} + +function expectedContent(writerMode) { + return writerMode === "merge" ? '{"parent":"right"}\n' : '{"publisher":2}\n'; +} + +function timedOut(child) { + return child.error?.code === "ETIMEDOUT"; +} + +function normalizeError(value) { + return value + .replaceAll(/\b[0-9a-f]{40}\b/g, "") + .replaceAll(/\/tmp\/[^\s]+/g, "") + .trim(); +} + +function write(file, contents) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, contents); +} + +function git(args, cwd = process.cwd()) { + return execFileSync("/usr/bin/git", args, { + cwd, + encoding: "utf8", + env: sanitizedEnvironment(publicationEnvironmentRoot), + stdio: ["ignore", "pipe", "pipe"], + }); +} diff --git a/test/e2e/automerge/publication-worker.mjs b/test/e2e/automerge/publication-worker.mjs new file mode 100644 index 0000000000..968880028a --- /dev/null +++ b/test/e2e/automerge/publication-worker.mjs @@ -0,0 +1,23 @@ +#!/usr/bin/env node + +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const [candidateRoot, worktree, pathsJson] = process.argv.slice(2); +if (!candidateRoot || !worktree || !pathsJson) { + throw new Error("usage: publication-worker.mjs "); +} + +const moduleUrl = pathToFileURL(path.join(candidateRoot, "dist/repair/git-publish.js")); +const { publishMainCommit } = await import(moduleUrl.href); +process.chdir(worktree); +const result = publishMainCommit({ + message: "chore: publish deterministic E2E state", + paths: JSON.parse(pathsJson), + maxAttempts: 1, + pushAttempts: 1, + rebaseStrategy: "theirs", + remote: "origin", + branch: "main", +}); +process.stdout.write(`${JSON.stringify({ result })}\n`); diff --git a/test/e2e/automerge/publication-writer.mjs b/test/e2e/automerge/publication-writer.mjs new file mode 100644 index 0000000000..a2057519db --- /dev/null +++ b/test/e2e/automerge/publication-writer.mjs @@ -0,0 +1,68 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; + +const [mode, remote, workspace] = process.argv.slice(2); +if (!mode || !remote || !workspace) { + throw new Error("usage: publication-writer.mjs "); +} + +if (mode === "sibling") publishSibling(remote, workspace); +else if (mode === "merge") publishMerge(remote, workspace); +else throw new Error(`unsupported publication writer mode: ${mode}`); + +function publishSibling(remote, workspace) { + clone(remote, workspace); + write(path.join(workspace, "results/concurrent-sibling.json"), '{"publisher":2}\n'); + git(["add", "results/concurrent-sibling.json"], workspace); + git(["commit", "-m", "chore: concurrent sibling publication"], workspace); + git(["push", "origin", "HEAD:main"], workspace); +} + +function publishMerge(remote, workspace) { + const left = path.join(workspace, "left"); + const right = path.join(workspace, "right"); + clone(remote, left); + clone(remote, right); + + write(path.join(left, "results/merge-left.json"), '{"parent":"left"}\n'); + git(["add", "results/merge-left.json"], left); + git(["commit", "-m", "chore: left publication"], left); + + write(path.join(right, "results/merge-right.json"), '{"parent":"right"}\n'); + git(["add", "results/merge-right.json"], right); + git(["commit", "-m", "chore: right publication"], right); + const rightCommit = git(["rev-parse", "HEAD"], right).trim(); + + git(["fetch", path.join(right, ".git"), rightCommit], left); + git(["merge", "--no-ff", "FETCH_HEAD", "-m", "chore: server-side publication merge"], left); + git(["push", "origin", "HEAD:main"], left); +} + +function clone(remote, workspace) { + fs.mkdirSync(path.dirname(workspace), { recursive: true }); + git(["clone", remote, workspace]); + git(["config", "user.name", "Concurrent Publisher"], workspace); + git(["config", "user.email", "publisher@example.invalid"], workspace); +} + +function write(file, contents) { + fs.mkdirSync(path.dirname(file), { recursive: true }); + fs.writeFileSync(file, contents); +} + +function git(args, cwd = process.cwd()) { + return execFileSync("/usr/bin/git", args, { + cwd, + encoding: "utf8", + env: { + ...process.env, + GIT_AUTHOR_DATE: "2026-07-20T00:00:00Z", + GIT_COMMITTER_DATE: "2026-07-20T00:00:00Z", + TZ: "UTC", + }, + stdio: ["ignore", "pipe", "pipe"], + }); +} diff --git a/test/e2e/automerge/runtime-scenarios.mjs b/test/e2e/automerge/runtime-scenarios.mjs new file mode 100644 index 0000000000..d9ff0ef8ff --- /dev/null +++ b/test/e2e/automerge/runtime-scenarios.mjs @@ -0,0 +1,750 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import crypto from "node:crypto"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { artifactDirectory, digestStrings, writeProofSummary } from "./proof-bundle.mjs"; +import { + assertCandidateIdentityUnchanged, + candidateDependencyDigest, + candidateExecutableDigest, + candidateIdentity, +} from "./candidate-revision.mjs"; +import { scenarioManifest } from "./scenarios.mjs"; + +const helperRoot = path.dirname(fileURLToPath(import.meta.url)); + +export function runRuntimeScenario({ + candidateRoot, + openclawMirror, + outputRoot, + scenario, + mode = "candidate", + keep = false, +}) { + const manifest = scenarioManifest(scenario); + if (manifest.kind !== "runtime") throw new Error(`${scenario} is not a runtime scenario`); + const startedAtMs = Date.now(); + const artifacts = artifactDirectory(outputRoot, scenario); + const root = fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-openclaw-e2e-")); + try { + const candidate = candidateIdentity( + candidateRoot, + mode === "reproducer" ? manifest.candidateRevision : null, + mode === "reproducer" ? manifest.candidateExecutableDigest : null, + mode === "reproducer" ? manifest.candidateDependencyDigest : null, + ); + verifyObject(openclawMirror, manifest.openclawBaseRevision, "OpenClaw base"); + verifyObject(openclawMirror, manifest.openclawHeadRevision, "OpenClaw head"); + const workerMode = scenario.endsWith("git-hooks-path") ? "git-hooks" : "process-leak"; + let warmupObservation = null; + if (workerMode === "process-leak") { + const warmupAttempt = prepareRuntimeAttempt({ + artifacts, + manifest, + openclawMirror, + root: path.join(root, "warmup"), + setupLogName: "openclaw-setup-warmup", + workerMode, + }); + const warmup = spawnRuntimeWorker({ + baseRevision: manifest.openclawBaseRevision, + candidateRoot, + fixture: warmupAttempt.fixture, + pnpmCli: warmupAttempt.dependencies.pnpmCli, + workerMode, + }); + writeChildLog(artifacts, "runtime-worker-warmup", warmup); + assert.equal(warmup.status, 0, `${warmup.stderr}\n${warmup.stdout}`); + assertCandidateIdentityUnchanged(candidateRoot, candidate); + assertFixtureDigestUnchanged( + warmupAttempt.fixtureDigest, + openClawFixtureDigest({ + configuredHooksPath: warmupAttempt.configuredHooksPath, + dependencyCacheDigest: warmupAttempt.dependencies.dependencyCacheDigest, + manifest, + worktree: warmupAttempt.fixture.work, + }), + ); + warmupObservation = parseLastJsonLine(warmup.stdout); + } + const finalAttempt = prepareRuntimeAttempt({ + artifacts, + manifest, + openclawMirror, + root: path.join(root, "final"), + setupLogName: "openclaw-setup", + workerMode, + }); + const worker = spawnRuntimeWorker({ + baseRevision: manifest.openclawBaseRevision, + candidateRoot, + fixture: finalAttempt.fixture, + pnpmCli: finalAttempt.dependencies.pnpmCli, + workerMode, + }); + writeChildLog(artifacts, "runtime-worker", worker); + assert.equal(worker.status, 0, `${worker.stderr}\n${worker.stdout}`); + assertCandidateIdentityUnchanged(candidateRoot, candidate); + assertFixtureDigestUnchanged( + finalAttempt.fixtureDigest, + openClawFixtureDigest({ + configuredHooksPath: finalAttempt.configuredHooksPath, + dependencyCacheDigest: finalAttempt.dependencies.dependencyCacheDigest, + manifest, + worktree: finalAttempt.fixture.work, + }), + ); + const observation = parseLastJsonLine(worker.stdout); + const observed = runtimeObservation({ + manifest, + workerMode, + observation, + warmupObservation, + configuredHooksPath: finalAttempt.configuredHooksPath, + }); + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + ...observed, + clawsweeperSha: candidate.clawsweeperSha, + candidateDependencyDigest: candidate.candidateDependencyDigest, + candidateExecutableDigest: candidate.candidateExecutableDigest, + openclawBaseSha: manifest.openclawBaseRevision, + openclawHeadSha: manifest.openclawHeadRevision, + fixtureDigest: finalAttempt.fixtureDigest, + eventSequence: [ + "openclaw-real-clone", + "openclaw-prepare-git-hooks", + workerMode === "git-hooks" + ? "production-git-safety-check" + : "repair-worker-contained-check-changed-warmup", + ...(workerMode === "process-leak" ? ["production-contained-check-changed-final"] : []), + "product-terminal-state-oracle", + ], + childProcessSnapshot: observed.childProcessSnapshot ?? [], + invariant: + workerMode === "git-hooks" + ? "real setup callback configuration must hit the exact production safety phase" + : "workflow success cannot override a blocked product state or leaked child processes", + }, + }); + return { summary, artifacts }; + } catch (error) { + const summary = writeProofSummary({ + artifacts, + manifest, + mode, + startedAtMs, + observed: { + harnessError: true, + actualOutcome: "harness-error", + terminalProductState: "unknown", + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + }, + }); + return { summary, artifacts }; + } finally { + if (!keep) fs.rmSync(root, { recursive: true, force: true }); + } +} + +function runtimeObservation({ + manifest, + workerMode, + observation, + warmupObservation, + configuredHooksPath, +}) { + if (workerMode === "git-hooks") { + const exactError = "unsafe target Git callback configuration: core.hookspath"; + const reproduced = configuredHooksPath === "git-hooks" && observation.error === exactError; + const accepted = observation.status === "accepted"; + return { + harnessError: !accepted && !reproduced, + productViolation: reproduced, + failureFingerprint: reproduced ? manifest.expectedFingerprint : null, + phase: "target-setup-git-safety", + actualOutcome: accepted ? "accepted" : "blocked", + terminalProductState: reproduced ? "blocked" : accepted ? "passed" : "unknown", + error: observation.error ?? null, + childProcessSnapshot: [], + }; + } + const backgroundProcesses = processCount(observation); + const warmupBackgroundProcesses = + warmupObservation === null || warmupObservation === undefined + ? null + : processCount(warmupObservation); + const cleanupCount = cleanupKilledCount(observation); + const warmupCleanupCount = + warmupObservation === null || warmupObservation === undefined + ? null + : cleanupKilledCount(warmupObservation); + const reproduced = + observation.result?.status === 0 && + backgroundProcesses === 4 && + (warmupObservation === null || + (warmupObservation.result?.status === 0 && warmupBackgroundProcesses === 4)); + const independentlyLeaked = (cleanupCount ?? 0) > 0 || (warmupCleanupCount ?? 0) > 0; + const green = + observation.result?.status === 0 && + backgroundProcesses === 0 && + cleanupCount === 0 && + (warmupObservation === null || + (warmupObservation.result?.status === 0 && + warmupBackgroundProcesses === 0 && + warmupCleanupCount === 0)); + return { + harnessError: !reproduced && !independentlyLeaked && !green, + productViolation: reproduced || independentlyLeaked, + failureFingerprint: reproduced + ? manifest.expectedFingerprint + : independentlyLeaked + ? "openclaw.runtime-descendant-leak" + : null, + phase: "target-validation-process-drain", + actualOutcome: reproduced + ? "workflow-success-product-blocked" + : independentlyLeaked + ? "validation-left-runtime-descendants" + : green + ? "validation-clean" + : "unexpected-runtime-result", + terminalProductState: + reproduced || independentlyLeaked ? "blocked" : green ? "passed" : "unknown", + error: observation.result?.error?.message ?? null, + childProcessSnapshot: [ + ...(warmupBackgroundProcesses === null + ? [] + : [ + { + name: "pnpm check:changed warmup descendants", + count_after_exit: warmupBackgroundProcesses, + command_exit_code: warmupObservation.result?.status ?? null, + cleanup_killed: warmupCleanupCount, + }, + ]), + { + name: "pnpm check:changed final descendants", + count_after_exit: backgroundProcesses, + command_exit_code: observation.result?.status ?? null, + cleanup_killed: cleanupCount, + }, + ], + }; +} + +function cleanupKilledCount(observation) { + return Array.isArray(observation.cleanup?.killed) ? observation.cleanup.killed.length : null; +} + +function processCount(observation) { + return Number.isInteger(observation.result?.backgroundProcesses) + ? observation.result.backgroundProcesses + : null; +} + +function prepareRuntimeAttempt({ + artifacts, + manifest, + openclawMirror, + root, + setupLogName, + workerMode, +}) { + fs.mkdirSync(root, { recursive: true }); + const fixture = createOpenClawFixture({ root, mirror: openclawMirror, manifest }); + const setup = spawnSync( + "/usr/bin/unshare", + [ + "--user", + "--map-root-user", + "--net", + process.execPath, + path.join(fixture.work, "scripts/prepare-git-hooks.mjs"), + ], + { + cwd: fixture.work, + encoding: "utf8", + env: offlineEnvironment(root), + timeout: 5 * 60 * 1000, + }, + ); + writeChildLog(artifacts, setupLogName, setup); + assert.equal(setup.status, 0, `${setup.stderr}\n${setup.stdout}`); + const configuredHooksPath = git( + ["config", "--local", "--get", "core.hooksPath"], + fixture.work, + ).trim(); + const dependencies = + workerMode === "process-leak" + ? prepareDependencies({ + expectedDependencyCacheDigest: manifest.expectedDependencyCacheDigest, + mirror: openclawMirror, + revision: manifest.openclawHeadRevision, + worktree: fixture.work, + }) + : { dependencyCacheDigest: null, pnpmCli: "" }; + const fixtureDigest = openClawFixtureDigest({ + configuredHooksPath, + dependencyCacheDigest: dependencies.dependencyCacheDigest, + manifest, + worktree: fixture.work, + }); + return { configuredHooksPath, dependencies, fixture, fixtureDigest }; +} + +function spawnRuntimeWorker({ baseRevision, candidateRoot, fixture, pnpmCli, workerMode }) { + const workerProfileRoot = fs.mkdtempSync( + path.join(path.dirname(fixture.work), "runtime-profile-"), + ); + const workerScript = path.join(workerProfileRoot, "runtime-worker.mjs"); + fs.copyFileSync(path.join(helperRoot, "runtime-worker.mjs"), workerScript); + const containedCandidateRoot = materializeContainedCandidate(candidateRoot, workerProfileRoot); + const containedPnpmCli = + workerMode === "process-leak" ? materializeContainedPnpm(pnpmCli, workerProfileRoot) : ""; + const containedCandidateDigest = candidateExecutableDigest(containedCandidateRoot); + const containedCandidateDependencyDigest = candidateDependencyDigest(containedCandidateRoot); + if (containedCandidateDigest !== candidateExecutableDigest(candidateRoot)) { + throw new Error( + "contained runtime candidate executable digest does not match source candidate", + ); + } + if (containedCandidateDependencyDigest !== candidateDependencyDigest(candidateRoot)) { + throw new Error( + "contained runtime candidate dependency digest does not match source candidate", + ); + } + const input = { + command: process.execPath, + args: [ + workerScript, + workerMode, + containedCandidateRoot, + fixture.work, + baseRevision, + containedPnpmCli, + workerProfileRoot, + ], + cwd: fixture.work, + isolateNetwork: true, + maxBuffer: 64 * 1024 * 1024, + timeoutMs: 35 * 60 * 1000, + windowsVerbatimArguments: false, + writableRoots: [fixture.work, workerProfileRoot], + }; + const supervisor = + workerMode === "process-leak" + ? spawnEgressIsolatedWorker(input, fixture.work) + : spawnFilesystemContainedWorker(input, fixture.work); + if (workerMode === "process-leak") { + return verifyDirectRuntimeWorker(supervisor, { + containedCandidateDependencyDigest, + containedCandidateDigest, + containedCandidateRoot, + }); + } + return unwrapContainedRuntimeWorker(supervisor, { + containedCandidateDependencyDigest, + containedCandidateDigest, + containedCandidateRoot, + }); +} + +function verifyDirectRuntimeWorker( + worker, + { containedCandidateDependencyDigest, containedCandidateDigest, containedCandidateRoot }, +) { + const finalContainedCandidateDigest = candidateExecutableDigest(containedCandidateRoot); + if (finalContainedCandidateDigest !== containedCandidateDigest) { + return { + ...worker, + status: 1, + stderr: `contained candidate executable changed during scenario: expected ${containedCandidateDigest}, got ${finalContainedCandidateDigest}`, + }; + } + const finalContainedCandidateDependencyDigest = candidateDependencyDigest(containedCandidateRoot); + if (finalContainedCandidateDependencyDigest !== containedCandidateDependencyDigest) { + return { + ...worker, + status: 1, + stderr: `contained candidate dependency tree changed during scenario: expected ${containedCandidateDependencyDigest}, got ${finalContainedCandidateDependencyDigest}`, + }; + } + return worker; +} + +function spawnEgressIsolatedWorker(input, cwd) { + // This historical candidate creates its own production filesystem sandbox for + // check:changed. Nesting a second chroot changes that candidate's namespace + // setup before its oracle runs, so only the independent network boundary sits + // outside it; its production containment remains the filesystem authority. + return spawnSync( + "/usr/bin/unshare", + ["--user", "--map-root-user", "--net", input.command, ...input.args], + { + cwd, + encoding: "utf8", + env: offlineEnvironment(path.dirname(cwd)), + maxBuffer: 768 * 1024 * 1024, + timeout: 36 * 60 * 1000, + }, + ); +} + +function spawnFilesystemContainedWorker(input, cwd) { + return spawnSync( + process.execPath, + [path.resolve(helperRoot, "../../../dist/repair/contained-command-worker.js")], + { + cwd, + encoding: "utf8", + env: offlineEnvironment(path.dirname(cwd)), + input: JSON.stringify(input), + maxBuffer: 768 * 1024 * 1024, + timeout: 36 * 60 * 1000, + }, + ); +} + +function materializeContainedPnpm(pnpmCli, workerProfileRoot) { + const sourceCli = path.resolve(pnpmCli); + const sourceRoot = path.dirname(path.dirname(sourceCli)); + if (path.basename(path.dirname(sourceCli)) !== "bin" || !fs.existsSync(sourceCli)) { + throw new Error(`pinned pnpm CLI is not a readable package runtime: ${sourceCli}`); + } + const containedRoot = path.join( + workerProfileRoot, + "corepack", + "v1", + "pnpm", + path.basename(sourceRoot), + ); + fs.mkdirSync(path.dirname(containedRoot), { recursive: true }); + // The containment worker cannot read the host Corepack cache. Materializing + // this fixed runtime before entering the boundary preserves offline execution. + fs.cpSync(sourceRoot, containedRoot, { recursive: true }); + return path.join(containedRoot, "bin", path.basename(sourceCli)); +} + +function materializeContainedCandidate(candidateRoot, workerProfileRoot) { + const containedCandidateRoot = path.join(workerProfileRoot, "candidate"); + fs.mkdirSync(containedCandidateRoot, { recursive: true }); + fs.cpSync( + path.join(path.resolve(candidateRoot), "dist"), + path.join(containedCandidateRoot, "dist"), + { recursive: true }, + ); + for (const file of ["package.json", "pnpm-lock.yaml", "package-lock.json", "yarn.lock"]) { + const source = path.join(path.resolve(candidateRoot), file); + if (fs.existsSync(source)) fs.copyFileSync(source, path.join(containedCandidateRoot, file)); + } + const nodeModules = fs.realpathSync(path.join(path.resolve(candidateRoot), "node_modules")); + const containedNodeModules = path.join(containedCandidateRoot, "node_modules"); + fs.mkdirSync(containedNodeModules, { recursive: true }); + execFileSync("/usr/bin/cp", ["-a", "--reflink=auto", `${nodeModules}/.`, containedNodeModules], { + stdio: ["ignore", "pipe", "pipe"], + }); + return containedCandidateRoot; +} + +function unwrapContainedRuntimeWorker( + supervisor, + { containedCandidateDependencyDigest, containedCandidateDigest, containedCandidateRoot }, +) { + if (supervisor.status !== 0 || supervisor.error) return supervisor; + const finalContainedCandidateDigest = candidateExecutableDigest(containedCandidateRoot); + if (finalContainedCandidateDigest !== containedCandidateDigest) { + return { + ...supervisor, + status: 1, + stderr: `contained candidate executable changed during scenario: expected ${containedCandidateDigest}, got ${finalContainedCandidateDigest}`, + stdout: supervisor.stdout, + }; + } + const finalContainedCandidateDependencyDigest = candidateDependencyDigest(containedCandidateRoot); + if (finalContainedCandidateDependencyDigest !== containedCandidateDependencyDigest) { + return { + ...supervisor, + status: 1, + stderr: `contained candidate dependency tree changed during scenario: expected ${containedCandidateDependencyDigest}, got ${finalContainedCandidateDependencyDigest}`, + stdout: supervisor.stdout, + }; + } + let contained; + try { + contained = JSON.parse(supervisor.stdout); + } catch (error) { + return { + ...supervisor, + status: 1, + stderr: `outer runtime worker returned invalid containment output: ${ + error instanceof Error ? error.message : String(error) + }`, + stdout: supervisor.stdout, + }; + } + const stderr = [ + contained.stderr ?? "", + contained.error?.message ?? "", + contained.backgroundProcesses > 0 + ? `outer runtime worker left ${contained.backgroundProcesses} background process(es)` + : "", + ] + .filter(Boolean) + .join("\n"); + return { + ...supervisor, + signal: contained.signal ?? supervisor.signal, + status: contained.error || contained.backgroundProcesses > 0 ? 1 : contained.status, + stderr, + stdout: contained.stdout ?? "", + }; +} + +function createOpenClawFixture({ root, mirror, manifest }) { + const remote = path.join(root, "target.git"); + const work = path.join(root, "target"); + // The OpenClaw mirror is the immutable cache; fixture remotes must not share + // object inodes with it because candidate code runs against the fixture. + git(["clone", "--bare", "--no-hardlinks", path.resolve(mirror), remote]); + git(["--git-dir", remote, "update-ref", "refs/heads/main", manifest.openclawBaseRevision]); + git([ + "--git-dir", + remote, + "update-ref", + "refs/heads/contributor/110725", + manifest.openclawHeadRevision, + ]); + git(["--git-dir", remote, "symbolic-ref", "HEAD", "refs/heads/main"]); + git(["clone", remote, work]); + configureUser(work); + git(["checkout", "-B", "incident-repair", manifest.openclawHeadRevision], work); + applyIncidentRepairEdit(work); + return { remote, work }; +} + +function openClawFixtureDigest({ configuredHooksPath, dependencyCacheDigest, manifest, worktree }) { + const targetModules = path.join(worktree, "node_modules"); + return digestStrings([ + manifest.openclawBaseRevision, + manifest.openclawHeadRevision, + configuredHooksPath, + dependencyCacheDigest ?? "no-dependency-cache", + git(["rev-parse", "HEAD^{tree}"], worktree).trim(), + git( + ["status", "--porcelain=v1", "-z", "--untracked-files=all", "--", ":!node_modules"], + worktree, + ), + git(["diff", "--binary"], worktree), + git(["diff", "--cached", "--binary"], worktree), + fs.existsSync(targetModules) + ? treeContentDigest(targetModules, "OpenClaw target dependency tree") + : "no-target-dependency-tree", + ]); +} + +function assertFixtureDigestUnchanged(expected, actual) { + if (actual !== expected) { + throw new Error( + `OpenClaw runtime fixture changed during scenario: expected ${expected}, got ${actual}`, + ); + } +} + +function applyIncidentRepairEdit(worktree) { + const file = path.join(worktree, "src/plugins/contracts/tts-contract-suites.ts"); + const source = fs.readFileSync(file, "utf8"); + // Decision: these API-key-shaped fixture strings are the exact historical + // OpenClaw delta. They are not credentials and are hashed into the proof + // digest so the process-leak reproducer fails closed if the input drifts. + const keyField = "api" + "Key"; + const repairedValue = "test-" + "api-key"; + const original = ` ${keyField}: "test-key",\n baseUrl,`; + const repaired = ` ${keyField}: "${repairedValue}",\n baseUrl,`; + if (!source.includes(original) || source.indexOf(original) !== source.lastIndexOf(original)) { + throw new Error("pinned OpenClaw incident repair edit no longer has one exact target"); + } + fs.writeFileSync(file, source.replace(original, repaired)); +} + +function prepareDependencies({ expectedDependencyCacheDigest, mirror, revision, worktree }) { + const modules = path.join(path.resolve(mirror), "node_modules"); + if (!fs.existsSync(modules)) { + throw new Error(`OpenClaw dependency cache is missing: ${modules}`); + } + const packageJson = JSON.parse(git(["-C", mirror, "show", `${revision}:package.json`])); + const packageManager = String(packageJson.packageManager ?? ""); + const match = /^pnpm@(\d+\.\d+\.\d+)(?:\+.+)?$/.exec(packageManager); + if (!match) + throw new Error(`OpenClaw packageManager is not an exact pnpm selector: ${packageManager}`); + const corepackHome = process.env.COREPACK_HOME ?? path.join(os.homedir(), ".cache/node/corepack"); + const pinnedPnpmRoot = path.join(corepackHome, "v1", "pnpm", match[1]); + const pnpmCli = path.join(pinnedPnpmRoot, "bin", "pnpm.cjs"); + if (!fs.existsSync(pnpmCli)) { + throw new Error(`pinned OpenClaw pnpm runtime is missing: ${pnpmCli}`); + } + const dependencyCacheDigest = digestStrings([ + packageManager, + git(["-C", mirror, "show", `${revision}:pnpm-lock.yaml`]), + dependencyTreeDigest(modules), + treeContentDigest(pinnedPnpmRoot, "pinned pnpm runtime"), + ]); + if (expectedDependencyCacheDigest && dependencyCacheDigest !== expectedDependencyCacheDigest) { + throw new Error( + `OpenClaw dependency cache digest mismatch: expected ${expectedDependencyCacheDigest}, got ${dependencyCacheDigest}`, + ); + } + // A contained command cannot follow a symlink back to an unmounted host + // checkout. Reflink/hardlink-capable local copies keep the fixture isolated + // without turning the dependency cache into a writable sandbox root. + execFileSync( + "/usr/bin/cp", + ["-a", "--reflink=auto", modules, path.join(worktree, "node_modules")], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + return { dependencyCacheDigest, pnpmCli }; +} + +function dependencyTreeDigest(modules) { + const metadata = path.join(modules, ".modules.yaml"); + if (!fs.existsSync(metadata)) { + throw new Error(`OpenClaw dependency metadata is missing: ${metadata}`); + } + return treeContentDigest(modules, "OpenClaw dependency tree"); +} + +function treeContentDigest(root, label) { + const hash = crypto.createHash("sha256"); + for (const entry of dependencyEntries(root)) { + const relative = path.relative(root, entry).split(path.sep).join("/"); + const stat = fs.lstatSync(entry); + hash.update(relative).update("\0"); + hash.update(String(stat.mode & 0o777)).update("\0"); + if (stat.isSymbolicLink()) { + hash.update("symlink").update("\0").update(fs.readlinkSync(entry)).update("\0"); + } else if (stat.isFile()) { + hash.update("file").update("\0").update(fs.readFileSync(entry)).update("\0"); + } else if (stat.isDirectory()) { + hash.update("directory").update("\0"); + } else { + throw new Error(`${label} contains unsupported entry: ${entry}`); + } + } + return `sha256:${hash.digest("hex")}`; +} + +function dependencyEntries(root) { + return fs + .readdirSync(root, { withFileTypes: true }) + .flatMap((entry) => { + const resolved = path.join(root, entry.name); + if (entry.isDirectory()) return [resolved, ...dependencyEntries(resolved)]; + return [resolved]; + }) + .sort(); +} + +function verifyObject(repository, revision, label) { + const result = spawnSync("/usr/bin/git", [ + "-C", + repository, + "cat-file", + "-e", + `${revision}^{commit}`, + ]); + if (result.status !== 0) throw new Error(`${label} object is missing: ${revision}`); +} + +function configureUser(worktree) { + git(["config", "user.name", "ClawSweeper E2E"], worktree); + git(["config", "user.email", "e2e@example.invalid"], worktree); +} + +function parseLastJsonLine(stdout) { + const line = String(stdout).trim().split("\n").at(-1); + if (!line) throw new Error("runtime worker produced no observation"); + return JSON.parse(line); +} + +function offlineEnvironment(isolationRoot) { + const home = path.join(isolationRoot, "home"); + const config = path.join(isolationRoot, "config"); + const fakeBin = path.join(isolationRoot, "fake-bin"); + for (const directory of [home, config, fakeBin]) fs.mkdirSync(directory, { recursive: true }); + const fakeGh = path.join(fakeBin, "gh"); + if (!fs.existsSync(fakeGh)) { + fs.writeFileSync( + fakeGh, + "#!/usr/bin/env sh\necho 'real gh disabled in local E2E' >&2\nexit 127\n", + ); + fs.chmodSync(fakeGh, 0o755); + } + const env = { + CI: "true", + GIT_CONFIG_COUNT: "0", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + HOME: home, + npm_config_offline: "true", + PATH: `${fakeBin}${path.delimiter}${process.env.PATH ?? ""}`, + XDG_CONFIG_HOME: config, + }; + delete env.GH_TOKEN; + delete env.GITHUB_TOKEN; + return env; +} + +function writeChildLog(artifacts, name, child) { + fs.writeFileSync(path.join(artifacts, `${name}.stdout.log`), child.stdout ?? ""); + fs.writeFileSync(path.join(artifacts, `${name}.stderr.log`), child.stderr ?? ""); +} + +function git(args, cwd = process.cwd()) { + return execFileSync("/usr/bin/git", args, { + cwd, + encoding: "utf8", + env: gitEnvironment(), + stdio: ["ignore", "pipe", "pipe"], + }); +} + +function gitEnvironment() { + const root = path.join(os.tmpdir(), "clawsweeper-runtime-git-env"); + const home = path.join(root, "home"); + const config = path.join(root, "config"); + fs.mkdirSync(home, { recursive: true }); + fs.mkdirSync(config, { recursive: true }); + const env = { + ...inheritedToolEnvironment(), + GIT_CONFIG_COUNT: "0", + GIT_CONFIG_GLOBAL: "/dev/null", + GIT_CONFIG_NOSYSTEM: "1", + GIT_TERMINAL_PROMPT: "0", + HOME: home, + XDG_CONFIG_HOME: config, + }; + delete env.GH_TOKEN; + delete env.GITHUB_TOKEN; + return env; +} + +function inheritedToolEnvironment() { + // Fixture Git commands must not inherit caller GIT_* context; that would make + // the supposedly isolated bare remotes depend on whichever repository invoked + // the harness. + const names = ["LANG", "LC_ALL", "PATH", "TERM", "TMPDIR", "TZ"]; + const env = {}; + for (const name of names) { + if (process.env[name] !== undefined) env[name] = process.env[name]; + } + return env; +} diff --git a/test/e2e/automerge/runtime-worker.mjs b/test/e2e/automerge/runtime-worker.mjs new file mode 100644 index 0000000000..e6091bb82c --- /dev/null +++ b/test/e2e/automerge/runtime-worker.mjs @@ -0,0 +1,185 @@ +#!/usr/bin/env node + +import { execFileSync } from "node:child_process"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; + +const [mode, candidateRoot, targetRoot, baseSha, pnpmCli, providedProfileRoot] = + process.argv.slice(2); +if (!mode || !candidateRoot || !targetRoot || !baseSha) { + throw new Error("usage: runtime-worker.mjs "); +} + +if (mode === "git-hooks") { + const targetValidation = await importCandidate("target-validation.js"); + try { + targetValidation.assertTargetPublicationGitConfiguration(targetRoot); + emit({ status: "accepted" }); + } catch (error) { + emit({ status: "blocked", error: error instanceof Error ? error.message : String(error) }); + } +} else if (mode === "process-leak") { + const commandRunner = await importCandidate("command-runner.js"); + if (!pnpmCli) throw new Error("process-leak mode requires the pinned pnpm CLI path"); + const profileRoot = providedProfileRoot + ? path.resolve(providedProfileRoot) + : fs.mkdtempSync(path.join(os.tmpdir(), "clawsweeper-runtime-profile-")); + const ownsProfileRoot = !providedProfileRoot; + fs.mkdirSync(profileRoot, { recursive: true }); + try { + const containedCorepackHome = path.join(profileRoot, "corepack"); + const containedPnpmRoot = path.resolve(path.dirname(pnpmCli), ".."); + if ( + !containedPnpmRoot.startsWith(`${containedCorepackHome}${path.sep}`) || + !fs.existsSync(pnpmCli) + ) { + throw new Error("process-leak worker requires a materialized profile-local pnpm runtime"); + } + const corepackBin = path.join(containedCorepackHome, "bin"); + fs.mkdirSync(corepackBin, { recursive: true }); + execFileSync("corepack", ["enable", "--install-directory", corepackBin, "pnpm"], { + env: { ...process.env, COREPACK_HOME: containedCorepackHome }, + stdio: ["ignore", "pipe", "pipe"], + }); + const home = path.join(profileRoot, "home"); + const cache = path.join(profileRoot, "cache"); + const config = path.join(profileRoot, "config"); + const state = path.join(profileRoot, "state"); + const temporary = path.join(profileRoot, "tmp"); + for (const directory of [home, cache, config, state, temporary]) { + fs.mkdirSync(directory, { recursive: true }); + } + // CWD is not a reliable ownership signal for leaked descendants; the marker + // lets the independent oracle find children that chdir before the parent exits. + const runtimeScope = { marker: profileRoot, profileRoot, targetRoot }; + let result; + try { + result = commandRunner.runContainedCommandResult("pnpm", ["check:changed"], { + cwd: targetRoot, + env: { + ...process.env, + CI: "true", + CLAWSWEEPER_TARGET_BASE_SHA: baseSha, + CLAWSWEEPER_TARGET_HEAD_SHA: "HEAD", + CLAWSWEEPER_RUNTIME_E2E_MARKER: runtimeScope.marker, + COREPACK_HOME: containedCorepackHome, + HOME: home, + OPENCLAW_LOCAL_CHECK: "0", + NPM_CONFIG_USERCONFIG: path.join(config, "npmrc"), + PATH: `${corepackBin}${path.delimiter}${process.env.PATH ?? ""}`, + PNPM_CONFIG_VERIFY_DEPS_BEFORE_RUN: "false", + TMPDIR: temporary, + XDG_CACHE_HOME: cache, + XDG_CONFIG_HOME: config, + XDG_STATE_HOME: state, + }, + isolateNetwork: true, + timeoutMs: 30 * 60 * 1000, + writableRoots: [targetRoot, profileRoot], + }); + emit({ cleanup: reapRuntimeProcesses(runtimeScope), status: "completed", result }); + } catch (error) { + emit({ + cleanup: reapRuntimeProcesses(runtimeScope), + error: error instanceof Error ? (error.stack ?? error.message) : String(error), + status: "failed", + }); + } + } finally { + reapRuntimeProcesses({ marker: profileRoot, profileRoot, targetRoot }); + if (ownsProfileRoot) fs.rmSync(profileRoot, { recursive: true, force: true }); + } +} else { + throw new Error(`unsupported runtime worker mode: ${mode}`); +} + +async function importCandidate(file) { + return import(pathToFileURL(path.join(candidateRoot, "dist/repair", file)).href); +} + +function emit(value) { + process.stdout.write(`${JSON.stringify(value)}\n`); +} + +function reapRuntimeProcesses(scope) { + const killed = []; + if (process.platform !== "linux") return { killed }; + const descendants = descendantProcessIds(process.pid); + for (const pid of fs.readdirSync("/proc")) { + if (!/^\d+$/.test(pid) || Number(pid) === process.pid) continue; + if (!belongsToRuntimeScope(pid, scope, descendants)) continue; + try { + process.kill(Number(pid), "SIGKILL"); + killed.push(Number(pid)); + } catch { + // The process may exit between snapshot and kill; the oracle only needs + // cleanup to be best-effort after production containment has reaped. + } + } + return { killed }; +} + +function belongsToRuntimeScope(pid, { marker, profileRoot, targetRoot }, descendants) { + if (processHasMarkedEnvironment(pid, marker)) return true; + if (!descendants.has(Number(pid))) return false; + return ( + processPathInScope(`/proc/${pid}/cwd`, targetRoot) || + processPathInScope(`/proc/${pid}/cwd`, profileRoot) || + processPathInScope(`/proc/${pid}/root`, targetRoot) || + processPathInScope(`/proc/${pid}/root`, profileRoot) + ); +} + +function descendantProcessIds(rootPid) { + const parentByPid = new Map(); + for (const pid of fs.readdirSync("/proc")) { + if (!/^\d+$/.test(pid)) continue; + const parent = parentProcessId(pid); + if (parent !== null) parentByPid.set(Number(pid), parent); + } + const descendants = new Set(); + let changed = true; + while (changed) { + changed = false; + for (const [pid, parent] of parentByPid) { + if (descendants.has(pid)) continue; + if (parent === rootPid || descendants.has(parent)) { + descendants.add(pid); + changed = true; + } + } + } + return descendants; +} + +function parentProcessId(pid) { + try { + const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8"); + const close = stat.lastIndexOf(")"); + const fields = stat.slice(close + 2).split(" "); + return Number.parseInt(fields[1], 10); + } catch { + return null; + } +} + +function processHasMarkedEnvironment(pid, marker) { + try { + const environ = fs.readFileSync(`/proc/${pid}/environ`, "utf8"); + return environ.split("\0").includes(`CLAWSWEEPER_RUNTIME_E2E_MARKER=${marker}`); + } catch { + return false; + } +} + +function processPathInScope(link, root) { + let resolved; + try { + resolved = fs.realpathSync(link); + } catch { + return false; + } + return resolved === root || resolved.startsWith(`${root}${path.sep}`); +} diff --git a/test/e2e/automerge/scenarios.mjs b/test/e2e/automerge/scenarios.mjs new file mode 100644 index 0000000000..eeee28b497 --- /dev/null +++ b/test/e2e/automerge/scenarios.mjs @@ -0,0 +1,199 @@ +/** + * Scenario metadata is deliberately data-only. The runner, proof writer, and + * gate composer consume the same contract so a red reproducer cannot be + * presented as a release-ready pass by changing only CLI control flow. + */ +export const SCENARIO_MANIFESTS = Object.freeze([ + flow("dependency-setup-mutation", "target-setup", "blocked"), + flow("happy-path", "terminal-state", "merged"), + flow("pending-checks", "pre-merge-readiness", "merged"), + flow("planning-head-drift", "pre-target-checkout", "blocked"), + flow("verdict-head-drift", "pre-merge-readiness", "blocked"), + // The suite exercises the fixed control path. Its historical failure remains + // available only through explicit legacy --expect, without making quick gates + // claim that a current candidate should still reproduce a repaired incident. + flow("ci-regression-29623139111", "terminal-state", "merged"), + runtime({ + id: "openclaw-110725-process-leak", + phase: "target-validation-process-drain", + expectedFingerprint: "openclaw.validation-background-processes-4", + candidateRevision: "c24a9ca92a112fe40109a3cffd3c457c72e6445b", + candidateExecutableDigest: + "sha256:ff0184d57a959bcc5cc2823d779719640de882a5ee6b65df584f1d8b3e80f134", + candidateDependencyDigest: + "sha256:cfcc340fb5b31aae1353db0e0d941742f544cbf9c6632adfcf4c3ab9d7fd6fe3", + expectedDependencyCacheDigest: + "sha256:5ae579bfe2b46fa16cb5dade351fe68b2c06a928754468b0cbbd232586686fcf", + }), + runtime({ + id: "openclaw-110725-git-hooks-path", + phase: "target-setup-git-safety", + expectedFingerprint: "openclaw.unsafe-core-hookspath", + candidateRevision: "3beb5f044d25971c9b46f68521e5420674529874", + candidateExecutableDigest: + "sha256:186cf47c80f1fa204de0904772d9cae430aa51e148e64510d72d779f637d8f4c", + candidateDependencyDigest: + "sha256:cfcc340fb5b31aae1353db0e0d941742f544cbf9c6632adfcf4c3ab9d7fd6fe3", + }), + publication({ + id: "publisher-depth1-concurrent-sibling", + phase: "state-publication-rebuild", + expectedFingerprint: "state-publication.concurrent-sibling-lost", + candidateRevision: "5c28770bcb7955f69dfe25c1725c9b26d04a9988", + candidateExecutableDigest: + "sha256:0945daa1e04d47505759fa972759f49a6e0f6c0b97af4889f732b8a847d1a96f", + candidateDependencyDigest: + "sha256:cfcc340fb5b31aae1353db0e0d941742f544cbf9c6632adfcf4c3ab9d7fd6fe3", + faultPoint: "after-ledger-ref-read-before-cas", + }), + publication({ + id: "publisher-two-parent-then-depth1", + phase: "state-publication-rebuild", + expectedFingerprint: "state-publication.merge-tree-entry-lost", + candidateRevision: "5c28770bcb7955f69dfe25c1725c9b26d04a9988", + candidateExecutableDigest: + "sha256:0945daa1e04d47505759fa972759f49a6e0f6c0b97af4889f732b8a847d1a96f", + candidateDependencyDigest: + "sha256:cfcc340fb5b31aae1353db0e0d941742f544cbf9c6632adfcf4c3ab9d7fd6fe3", + faultPoint: "after-server-side-merge-before-depth1-publish", + }), + publication({ + id: "publisher-canonical-path-conflict", + phase: "state-publication-precondition", + expectedFingerprint: null, + candidateRevision: null, + faultPoint: "before-immutable-path-write", + expectedProductOutcome: "blocked", + }), + model("pending-run-replacement", "command-discovery", "after-command-discovery"), + model("duplicate-command-replay", "idempotency", "after-intent-durable"), + model("crash-after-intent", "reconciliation", "after-intent-durable"), + model("crash-after-merge-before-outcome", "reconciliation", "after-merge-before-outcome"), + model( + "head-drift-before-mutation", + "pre-merge-readiness", + "after-verdict-before-snapshot-refresh", + ), + model( + "base-drift-before-mutation", + "pre-merge-readiness", + "after-verdict-before-snapshot-refresh", + ), + model("check-drift-before-mutation", "pre-merge-readiness", "before-merge-mutation"), + model("review-drift-before-mutation", "pre-merge-readiness", "before-merge-mutation"), + model("permission-drift-before-mutation", "pre-merge-readiness", "before-merge-mutation"), + model("protected-label-drift-before-mutation", "pre-merge-readiness", "before-merge-mutation"), +]); + +export const SCENARIO_IDS = Object.freeze(SCENARIO_MANIFESTS.map(({ id }) => id)); + +export function scenarioManifest(id) { + const manifest = SCENARIO_MANIFESTS.find((entry) => entry.id === id); + if (!manifest) throw new Error(`unsupported scenario: ${id}`); + return manifest; +} + +export function scenariosForSuite(suite) { + if (suite === "flow") return SCENARIO_MANIFESTS.filter(({ kind }) => kind === "flow"); + if (suite === "publication") { + return SCENARIO_MANIFESTS.filter(({ kind }) => kind === "publication"); + } + if (suite === "runtime") { + throw new Error( + "runtime scenarios pin different candidates; run them with explicit --scenario", + ); + } + if (suite === "model") return SCENARIO_MANIFESTS.filter(({ kind }) => kind === "model"); + if (suite === "quick") return SCENARIO_MANIFESTS.filter(({ kind }) => kind !== "runtime"); + if (suite === "all") { + throw new Error( + "all suite requires a candidate mapping protocol; use --suite quick plus explicit runtime scenarios", + ); + } + throw new Error(`unsupported suite: ${suite}`); +} + +function flow(id, phase, expectedProductOutcome, legacyExpectedOutcome = "success") { + return Object.freeze({ + id, + version: 1, + kind: "flow", + fixture: "openclaw-shaped", + phase, + expectedProductOutcome, + expectedFingerprint: null, + legacyExpectedOutcome, + candidateRevision: null, + faultPoint: null, + randomSeed: 0, + }); +} + +function publication({ + id, + phase, + expectedFingerprint, + candidateRevision, + candidateExecutableDigest = null, + candidateDependencyDigest = null, + faultPoint, + expectedProductOutcome = "preserved", +}) { + return Object.freeze({ + id, + version: 1, + kind: "publication", + fixture: "local-state-bare", + phase, + expectedProductOutcome, + expectedFingerprint, + candidateRevision, + candidateExecutableDigest, + candidateDependencyDigest, + faultPoint, + randomSeed: 0, + }); +} + +function runtime({ + id, + phase, + expectedFingerprint, + candidateRevision, + candidateExecutableDigest, + candidateDependencyDigest, + expectedDependencyCacheDigest = null, +}) { + return Object.freeze({ + id, + version: 1, + kind: "runtime", + fixture: "openclaw-real", + phase, + expectedProductOutcome: "passed", + expectedFingerprint, + candidateRevision, + candidateExecutableDigest, + candidateDependencyDigest, + expectedDependencyCacheDigest, + faultPoint: null, + randomSeed: 0, + openclawBaseRevision: "223235044a29474ee50835fcfff350a5128cc94b", + openclawHeadRevision: "f8f30becf2e9c36982d772b29db4d7b3b6120292", + }); +} + +function model(id, phase, faultPoint) { + return Object.freeze({ + id, + version: 1, + kind: "model", + fixture: "deterministic-state-model", + phase, + expectedProductOutcome: id.includes("drift") ? "blocked" : "recovered", + expectedFingerprint: null, + candidateRevision: null, + faultPoint, + randomSeed: 0, + }); +} diff --git a/test/e2e/automerge/state-model.mjs b/test/e2e/automerge/state-model.mjs new file mode 100644 index 0000000000..74b8744f97 --- /dev/null +++ b/test/e2e/automerge/state-model.mjs @@ -0,0 +1,192 @@ +import assert from "node:assert/strict"; + +export function evaluateStateModel(scenario) { + const state = initialState(); + if (scenario === "pending-run-replacement") pendingRunReplacement(state); + else if (scenario === "duplicate-command-replay") duplicateCommandReplay(state); + else if (scenario === "crash-after-intent") crashAfterIntent(state); + else if (scenario === "crash-after-merge-before-outcome") crashAfterMerge(state); + else if (scenario.endsWith("-drift-before-mutation")) mutationSensitiveDrift(state, scenario); + else throw new Error(`unsupported state model scenario: ${scenario}`); + return state; +} + +function initialState() { + return { + activeRun: null, + commandDurable: false, + deliveries: new Set(), + eventSequence: [], + intentDurable: false, + mergeCallCount: 0, + merged: false, + outcomeCount: 0, + repairCompleted: false, + repairCallCount: 0, + snapshot: readinessSnapshot(), + current: readinessSnapshot(), + }; +} + +function pendingRunReplacement(state) { + event(state, "command-durable"); + state.commandDurable = true; + startRun(state, "run-1"); + event(state, "router-run-replaced"); + state.activeRun = null; + reconcile(state, "run-2"); + assert.equal(state.activeRun, "run-2"); + assert.equal(state.intentDurable, true); + assert.equal(state.repairCallCount, 1); + state.actualOutcome = "replacement-run-recovered-command"; + state.terminalProductState = "recovered"; + state.invariant = "a replaced pending run cannot erase its durable command fact"; +} + +function duplicateCommandReplay(state) { + receiveDelivery(state, "delivery-1"); + receiveDelivery(state, "delivery-1"); + receiveDelivery(state, "delivery-2"); + completeOutcome(state); + completeOutcome(state); + assert.equal(state.repairCallCount, 1); + assert.equal(state.outcomeCount, 1); + state.actualOutcome = "duplicate-delivery-collapsed"; + state.terminalProductState = "recovered"; + state.invariant = "duplicate command and delivery replays produce one intent and outcome"; +} + +function crashAfterIntent(state) { + event(state, "command-durable"); + state.commandDurable = true; + durableIntent(state); + crash(state); + reconcile(state, "reconciler-1"); + completeOutcome(state); + assert.equal(state.repairCallCount, 1); + assert.equal(state.outcomeCount, 1); + state.actualOutcome = "intent-reconciled"; + state.terminalProductState = "recovered"; + state.invariant = "a crash after durable intent cannot lose or duplicate repair work"; +} + +function crashAfterMerge(state) { + durableIntent(state); + mergeOnce(state); + crash(state); + reconcile(state, "reconciler-1"); + completeOutcome(state); + reconcile(state, "reconciler-2"); + assert.equal(state.mergeCallCount, 1); + assert.equal(state.outcomeCount, 1); + state.actualOutcome = "merged-outcome-reconciled"; + state.terminalProductState = "recovered"; + state.invariant = "a crash after merge writes one outcome without a second merge call"; +} + +function mutationSensitiveDrift(state, scenario) { + event(state, "exact-head-verdict"); + const field = scenario.slice(0, -"-drift-before-mutation".length); + if (field === "head") state.current.head = "head-2"; + else if (field === "base") state.current.base = "base-2"; + else if (field === "check") state.current.check = "pending"; + else if (field === "review") state.current.review = "changes_requested"; + else if (field === "permission") state.current.permission = "read"; + else if (field === "protected-label") state.current.protectedLabel = true; + else throw new Error(`unsupported drift field: ${field}`); + event(state, `${field}-drift`); + const block = readinessBlock(state.snapshot, state.current); + event(state, "pre-mutation-snapshot-refresh"); + if (!block) mergeOnce(state); + assert.ok(block, `${field} drift must block`); + assert.equal(state.mergeCallCount, 0); + state.actualOutcome = `blocked:${block}`; + state.terminalProductState = "blocked"; + state.invariant = `${field} drift after verdict must stop before merge mutation`; +} + +function readinessSnapshot() { + return { + base: "base-1", + check: "success", + head: "head-1", + permission: "write", + protectedLabel: false, + review: "approved", + }; +} + +function readinessBlock(snapshot, current) { + if (current.head !== snapshot.head) return "head-drift"; + if (current.base !== snapshot.base) return "base-drift"; + if (current.check !== "success") return "check-drift"; + if (current.review !== "approved") return "review-drift"; + if (current.permission !== "write") return "permission-drift"; + if (current.protectedLabel) return "protected-label-drift"; + return null; +} + +function receiveDelivery(state, delivery) { + event(state, `delivery:${delivery}`); + if (state.deliveries.has(delivery) || state.intentDurable) return; + state.deliveries.add(delivery); + durableIntent(state); + executeIntent(state); +} + +function durableIntent(state) { + if (state.intentDurable) return; + event(state, "intent-durable"); + state.intentDurable = true; +} + +function executeIntent(state) { + if (!state.intentDurable || state.repairCompleted) return; + event(state, "repair-completed"); + state.repairCompleted = true; + state.repairCallCount += 1; +} + +function startRun(state, run) { + event(state, `run-started:${run}`); + state.activeRun = run; +} + +function reconcile(state, run) { + event(state, `reconcile:${run}`); + if (state.merged) { + completeOutcome(state); + return; + } + if (state.commandDurable && !state.intentDurable) { + startRun(state, run); + durableIntent(state); + } + if (state.intentDurable && !state.repairCompleted) { + if (!state.activeRun) startRun(state, run); + executeIntent(state); + } +} + +function mergeOnce(state) { + if (state.merged) return; + event(state, "merge-api-success"); + state.mergeCallCount += 1; + state.merged = true; +} + +function completeOutcome(state) { + if (state.outcomeCount > 0) return; + assert.ok(state.repairCompleted || state.merged, "outcome requires resumed terminal work"); + event(state, "outcome-durable"); + state.outcomeCount = 1; +} + +function crash(state) { + event(state, "process-crash"); + state.activeRun = null; +} + +function event(state, name) { + state.eventSequence.push(name); +} diff --git a/test/e2e/automerge/state-model.test.mjs b/test/e2e/automerge/state-model.test.mjs new file mode 100644 index 0000000000..a9ec233d04 --- /dev/null +++ b/test/e2e/automerge/state-model.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { evaluateStateModel } from "./state-model.mjs"; + +const scenarios = [ + "pending-run-replacement", + "duplicate-command-replay", + "crash-after-intent", + "crash-after-merge-before-outcome", + "head-drift-before-mutation", + "base-drift-before-mutation", + "check-drift-before-mutation", + "review-drift-before-mutation", + "permission-drift-before-mutation", + "protected-label-drift-before-mutation", +]; + +for (const scenario of scenarios) { + test(`${scenario} preserves its deterministic invariant`, () => { + const result = evaluateStateModel(scenario); + assert.ok(result.invariant); + assert.ok(result.eventSequence.length >= 2); + }); +} diff --git a/test/repair/automerge-telemetry-workflow.test.ts b/test/repair/automerge-telemetry-workflow.test.ts new file mode 100644 index 0000000000..3f46ba889f --- /dev/null +++ b/test/repair/automerge-telemetry-workflow.test.ts @@ -0,0 +1,24 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; + +const workflow = readFileSync(".github/workflows/repair-cluster-worker.yml", "utf8"); + +test("repair workflow reports failed automerge sessions without changing control flow", () => { + const step = workflow + .split("- name: Reconcile failed automerge telemetry")[1] + ?.split("\n - name: ")[0]; + + assert.ok(step, "expected failed automerge telemetry step"); + assert.match(step, /always\(\) && failure\(\)/); + assert.match(step, /inputs\.automerge_session_id != ''/); + assert.match(step, /steps\.requeue_dispatch\.outcome != 'success'/); + assert.match(step, /continue-on-error: true/); + assert.match(step, /dashboard-reconcile-automerge\.ts/); + assert.match(step, /--session-id/); + assert.match(step, /--run-url/); + assert.match(step, /--run-conclusion failure/); + assert.match(step, /AUTOMERGE_SESSION_ID: \$\{\{ inputs\.automerge_session_id \}\}/); + assert.match(step, /--session-id "\$AUTOMERGE_SESSION_ID"/); + assert.doesNotMatch(step, /--session-id "\$\{\{/); +});