Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using System.Text.Json;
using CodeSpace.Core.Services.Supervisor;
using CodeSpace.Messages.Agents;
using CodeSpace.Messages.Agents.Benchmark;
using CodeSpace.Messages.Enums;
Expand Down Expand Up @@ -215,12 +216,20 @@ _ when detail.StartsWith($"{ModelCheckGateLabel}: ", StringComparison.Ordinal) =
/// rubric, a schema check with no schema) would invert the gate's fail-closed philosophy. Null = valid; else the
/// legible reason. The graders independently re-enforce every rule at grade time fail-closed, so a spec that
/// bypasses authoring validation (the supervisor lane, a raw API caller) still can never silently pass.
///
/// <para>The same holds for the grade window: the grader runs every step under (0,
/// <see cref="SupervisorLane.MaxAcceptanceGradeTimeoutSeconds"/>] whatever the contract says, so an authored
/// <see cref="SupervisorAcceptanceSpec.TimeoutSeconds"/> outside that range is refused here, where the operator can
/// see why, instead of being rewritten at grade time.</para>
/// </summary>
public static string? ValidateAuthored(SupervisorAcceptanceSpec spec)
{
if (spec.Command.All(string.IsNullOrWhiteSpace))
return "acceptance requires a non-empty command — the argv for TestsPass, the deliverable paths for every other kind.";

if (spec.TimeoutSeconds is { } window && (window <= 0 || window > SupervisorLane.MaxAcceptanceGradeTimeoutSeconds))
return $"acceptance timeoutSeconds must be between 1 and {SupervisorLane.MaxAcceptanceGradeTimeoutSeconds} (SupervisorLane.MaxAcceptanceGradeTimeoutSeconds, the longest any grade step runs); omit it to grade in the {SupervisorLane.AcceptanceGradeTimeoutSeconds}-second default.";

switch (spec.Kind)
{
case BenchmarkGradingKind.LlmJudge:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,16 @@ Task<BenchmarkGrade> GradePatchAsync(PatchAcceptanceGradeRequest request, Cancel
/// </summary>
Task<BenchmarkGrade> GradeAsync(Guid repositoryId, Guid teamId, string branch, SupervisorAcceptanceSpec spec, int timeoutSeconds, CancellationToken cancellationToken);

/// <summary>DC-4 slice 2 (the repo-less lane): grade the oracle DIRECTLY against an existing directory — the scratch workspace a repo-less run produced its declared deliverables in. The agent process has already exited, so grading its left-behind directory is equivalent to grading a clone of it; there is no git world to anchor an independent checkout on. Same per-kind oracles, same fail-closed posture.</summary>
/// <summary>
/// DC-4 slice 2 (the repo-less lane): grade the oracle DIRECTLY against an existing directory — the scratch
/// workspace a repo-less run produced its declared deliverables in. There is no git world to anchor an independent
/// checkout on, so this is NOT equivalent to grading a clone: the check runs in the agent's own live workspace, and
/// every byte the contract does not pin is the agent's. A declared <c>OraclePaths</c> digest pins only the literal
/// files it names; anything else the check executes or reads can decide its exit code — a module beside a pinned
/// script (a planted <c>json.py</c> flips a pinned <c>check.py</c>), test-runner config and plugins, manifest
/// scripts. The verdict is only as independent as an oracle that neither executes nor imports candidate-controlled
/// files. Same per-kind oracles, same fail-closed posture.
/// </summary>
Task<BenchmarkGrade> GradeDirectoryAsync(string directory, SupervisorAcceptanceSpec spec, Guid teamId, int timeoutSeconds, CancellationToken cancellationToken) =>
Task.FromResult(new BenchmarkGrade { Passed = false, Detail = "grade-error: directory grading is not supported by this grader", Class = Messages.Agents.Benchmark.GradeFailureClass.GraderFault });

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ public sealed class SupervisorAcceptanceGrader : ISupervisorAcceptanceGrader, IS
/// the SAME PR as any change to grading semantics — oracle dispatch, restore/tamper behavior, evidence
/// capture, fail-closed arms. Pinned by test; the literal is the wire value on durable receipts.
/// </summary>
public const string EvaluatorVersion = "supervisor-acceptance/v7"; // v7: delayed repository, patch, and captured-deliverable grades carry the producer's durable row/configured/observed identity into model-backed oracles; missing legacy evidence stays Unknown and is never inferred from the compatibility price label
public const string EvaluatorVersion = "supervisor-acceptance/v8"; // v8: every grade step (setup, check, oracle-restore git) runs under a bounded window — a non-positive authored timeout grades at the default instead of arming no wall clock, a longer one is capped at SupervisorLane.MaxAcceptanceGradeTimeoutSeconds

/// <summary>The grading clone + oracle commands run on the worker host's own local runner. NOT the deployment
/// default (<c>AgentDefaultRunnerSetting</c>): this funnel never reads a caller-supplied runner kind, and the
Expand Down Expand Up @@ -641,12 +641,23 @@ private static string Flatten(string paths)
Command = "git",
Args = args.ToList(),
WorkingDirectory = directory,
TimeoutSeconds = timeoutSeconds,
TimeoutSeconds = BoundedGradeWindow(timeoutSeconds),
};

/// <summary>
/// The window a grade step actually runs under, whatever the contract authored: a non-positive value grades at
/// <see cref="SupervisorLane.AcceptanceGradeTimeoutSeconds"/> — the runner reads it as "no wall clock at all", and
/// the batch run path has no stall watchdog behind it — and a longer one is capped at
/// <see cref="SupervisorLane.MaxAcceptanceGradeTimeoutSeconds"/>. Applied HERE, at the steps, because not every lane
/// validates the contract before grading: the supervisor's fold never calls <c>LocalAcceptanceVerifier.ValidateContract</c>.
/// </summary>
private static int BoundedGradeWindow(int timeoutSeconds) =>
timeoutSeconds <= 0 ? SupervisorLane.AcceptanceGradeTimeoutSeconds : Math.Min(timeoutSeconds, SupervisorLane.MaxAcceptanceGradeTimeoutSeconds);

private async Task<BenchmarkGrade> GradeWorkspaceAsync(WorkspaceGradeRequest request, CancellationToken cancellationToken)
{
var (directory, spec, teamId, timeoutSeconds, producerModel, protection) = request;
var (directory, spec, teamId, authoredTimeoutSeconds, producerModel, protection) = request;
var timeoutSeconds = BoundedGradeWindow(authoredTimeoutSeconds);

if (spec.SetupCommand is { Count: > 0 } setupCommand)
{
Expand Down
11 changes: 11 additions & 0 deletions backend/src/CodeSpace.Core/Services/Supervisor/SupervisorLane.cs
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,17 @@ public static class SupervisorLane
/// </summary>
public const int AcceptanceGradeTimeoutSeconds = 300;

/// <summary>
/// The LONGEST wall-clock window (seconds) any one step of an acceptance grade may run — the contract's setup, its
/// check, the oracle restore's git commands — whatever the contract authored. A contract's
/// <see cref="SupervisorAcceptanceSpec.TimeoutSeconds"/> only tunes the window inside (0, this]: a non-positive
/// value grades at <see cref="AcceptanceGradeTimeoutSeconds"/> (it used to arm no wall clock at all, so a grade
/// could run agent-written bytes for as long as they liked) and a longer one is capped here — an agent run's own
/// default budget. That rewrite is for lanes that never validate the contract; an operator contract authoring a
/// window outside the range is refused where it is authored (<c>AgentAcceptanceContract.ValidateAuthored</c>). Pinned (Rule 8).
/// </summary>
public const int MaxAcceptanceGradeTimeoutSeconds = 3600;

/// <summary>
/// P1.3 — the heartbeat interval a long SEQUENTIAL multi-target/multi-gate grade emits a ledger record at, so
/// the reconciler's staleness check (<see cref="StuckRunReconcilerService.LedgerLivenessWindow"/>, 5 min) never
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1696,8 +1696,8 @@ private async Task<BenchmarkGrade> GradeStopTargetsAsync(Guid teamId, IReadOnlyL
/// <summary>The verdict detail plus the oracle's integrity note, when there is one — the ONLY route a voided tamper or an unprotected judge has onto the durable stop outcome, which carries pass + detail and nothing else. No note ⇒ the detail verbatim (the dominant case stays byte-identical).</summary>
internal static string Annotated(string detail, string? oracleNote) => string.IsNullOrEmpty(oracleNote) ? detail : $"{detail} [{oracleNote}]";

/// <summary>The model-authored acceptance spec off a stop decision's payload (<see cref="SupervisorStopPayload.Acceptance"/> — its command + oracle Kind), best-effort (null when absent / malformed).</summary>
private static SupervisorAcceptanceSpec? ReadStopAcceptance(string payloadJson)
/// <summary>The model-authored acceptance spec off a stop decision's payload (<see cref="SupervisorStopPayload.Acceptance"/> — its command + oracle Kind), best-effort (null when absent / malformed). It reads through the payload's own acceptance slot, so a stored row never hands the stop gate a setup command or timeout (<see cref="ModelAuthoredAcceptanceConverter"/>). Internal so that rule is pinned on this reader rather than on a copy of it.</summary>
internal static SupervisorAcceptanceSpec? ReadStopAcceptance(string payloadJson)
{
try { return JsonSerializer.Deserialize<SupervisorStopPayload>(payloadJson, AgentJson.Options)?.Acceptance; }
catch (JsonException) { return null; }
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
using System.Text.Json;
using System.Text.Json.Serialization;

namespace CodeSpace.Messages.Agents;

/// <summary>
/// The wire rule of every acceptance slot a SUPERVISOR decision carries — a plan subtask's, a plan phase's, a stop's,
/// an amend proposal's replacement: the spec binds and persists WITHOUT <see cref="SupervisorAcceptanceSpec.SetupCommand"/>
/// and <see cref="SupervisorAcceptanceSpec.TimeoutSeconds"/>. Both are operator knobs. The setup argv runs workspace
/// bytes before the check; the timeout decides how long the grader runs anything at all. The decision schema never
/// offers either, but its <c>additionalProperties:false</c> is advisory — the server's schema check does not read it
/// and a schema-less fallback request carries none — so a reply naming them bound straight into the spec, was frozen
/// into the ledger, and the grader ran it.
///
/// <para>Declared on those PROPERTIES rather than on the type, so it holds wherever the payloads are read — the
/// decider's fresh bind, the projector's canonical bytes (and so the idempotency key), and every later re-read of a
/// stored ledger row, including one written before this rule existed — while an operator's own spec (node config,
/// <c>AgentTask.Acceptance</c>) keeps both. It strips on write as well, so no server path can freeze either knob into
/// a decision. Everything else the model authored binds unchanged: its check still grades the unit.</para>
/// </summary>
public sealed class ModelAuthoredAcceptanceConverter : JsonConverter<SupervisorAcceptanceSpec>
{
public override SupervisorAcceptanceSpec? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) =>
WithoutOperatorKnobs(JsonSerializer.Deserialize<SupervisorAcceptanceSpec>(ref reader, options));

public override void Write(Utf8JsonWriter writer, SupervisorAcceptanceSpec value, JsonSerializerOptions options) =>
JsonSerializer.Serialize(writer, WithoutOperatorKnobs(value), options);

private static SupervisorAcceptanceSpec? WithoutOperatorKnobs(SupervisorAcceptanceSpec? spec) =>
spec is null or { SetupCommand: null, TimeoutSeconds: null } ? spec : spec with { SetupCommand = null, TimeoutSeconds = null };
}
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,11 @@ public sealed record SupervisorAcceptanceSpec
/// P3.1: wall-clock cap (seconds) for THIS contract's grade, overriding the server's default (a plain compiled-in
/// constant — see the grading service). Absent ⇒ the default. A real test suite (a cold-cache dependency
/// install, a large monorepo) can author a longer window here instead of the check racing a one-size-fits-all
/// ceiling; a lightweight lint/artifact check can leave this unset.
/// ceiling; a lightweight lint/artifact check can leave this unset. Authoring refuses a value outside
/// [1, <c>SupervisorLane.MaxAcceptanceGradeTimeoutSeconds</c>] (<c>AgentAcceptanceContract.ValidateAuthored</c>); for a
/// lane that never validates, the grader bounds it the same way: a non-positive value grades at the default and a
/// longer one is capped. OPERATOR-only —
/// a supervisor decision's acceptance never carries it (<see cref="ModelAuthoredAcceptanceConverter"/>).
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public int? TimeoutSeconds { get; init; }
Expand All @@ -83,7 +87,9 @@ public sealed record SupervisorAcceptanceSpec
/// closed as an infrastructure fault (<c>AgentAcceptanceContract.IsInfraFailure</c>'s <c>setup-failed:</c>/
/// <c>setup-timed-out</c> details), never a statement about the code's correctness. Capped by the same
/// <see cref="TimeoutSeconds"/> window as the check itself (a separate budget was deliberately not added — the
/// contract author who needs a longer window for a cold-cache install already has one lever to pull).
/// contract author who needs a longer window for a cold-cache install already has one lever to pull). OPERATOR-only:
/// it executes workspace bytes before the check, so a supervisor decision's acceptance never carries it
/// (<see cref="ModelAuthoredAcceptanceConverter"/>).
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public IReadOnlyList<string>? SetupCommand { get; init; }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,10 @@ public sealed record SupervisorPlannedSubtask
/// noun as a stop / phase (<see cref="SupervisorAcceptanceSpec"/>). Null-omitted (<c>[JsonIgnore(WhenWritingNull)]</c>)
/// so a subtask without a contract serializes byte-identical to before. PURE DATA here: recorded + projected; the
/// per-unit acceptance GATE (grade each settled unit against this at the spawn fold) is a follow-up (slice 3).
/// Model-authored, so it never carries a setup command or timeout (<see cref="ModelAuthoredAcceptanceConverter"/>).
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonConverter(typeof(ModelAuthoredAcceptanceConverter))]
public SupervisorAcceptanceSpec? Acceptance { get; init; }

/// <summary>
Expand Down Expand Up @@ -266,8 +268,9 @@ public sealed record SupervisorAmendAcceptancePayload
/// <summary>True = forgo verification for this unit entirely; false = replace its oracle with <see cref="Acceptance"/>.</summary>
public bool Waive { get; init; }

/// <summary>The replacement oracle (full spec — kind, rubric/schema payloads, timeout). Null when <see cref="Waive"/> is true.</summary>
/// <summary>The replacement oracle (full spec — kind, rubric/schema payloads; never a setup command or timeout, see <see cref="ModelAuthoredAcceptanceConverter"/>). Null when <see cref="Waive"/> is true.</summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonConverter(typeof(ModelAuthoredAcceptanceConverter))]
public SupervisorAcceptanceSpec? Acceptance { get; init; }

/// <summary>Why the current oracle should not bind — quoted onto the human card, so the co-signer rules on evidence.</summary>
Expand Down Expand Up @@ -357,8 +360,10 @@ public static bool IsClarificationOutcome(string? outcome) =>
/// Optional model-authored OBJECTIVE acceptance for the terminal result — the L3→L4 "definition of done": a
/// server-run check the supervisor declares so "done" is a verified fact, not a self-report. Null-omitted
/// (<c>[JsonIgnore(WhenWritingNull)]</c>) so a stop WITHOUT acceptance serializes byte-identical to before —
/// the idempotency-key bytes are unchanged and exactly-once replay is unaffected. See <see cref="SupervisorAcceptanceSpec"/>.
/// the idempotency-key bytes are unchanged and exactly-once replay is unaffected. See <see cref="SupervisorAcceptanceSpec"/>;
/// model-authored, so it never carries a setup command or timeout (<see cref="ModelAuthoredAcceptanceConverter"/>).
/// </summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonConverter(typeof(ModelAuthoredAcceptanceConverter))]
public SupervisorAcceptanceSpec? Acceptance { get; init; }
}
3 changes: 2 additions & 1 deletion backend/src/CodeSpace.Messages/Agents/SupervisorPlanPhase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ public sealed record SupervisorPlanPhase
/// <summary>The plan-local subtask ids this phase groups (a subset of the plan's <see cref="SupervisorPlanPayload.Subtasks"/>). Empty for a descriptive-only phase.</summary>
public IReadOnlyList<string> SubtaskIds { get; init; } = Array.Empty<string>();

/// <summary>Optional per-phase OBJECTIVE acceptance (reuses the same noun as a stop's acceptance) — the server-runnable check this phase is "done" by. Recorded + projected in v1; the enforcing gate is a follow-up. Null-omitted so a phase without acceptance is byte-stable.</summary>
/// <summary>Optional per-phase OBJECTIVE acceptance (reuses the same noun as a stop's acceptance) — the server-runnable check this phase is "done" by. Recorded + projected in v1; the enforcing gate is a follow-up. Null-omitted so a phase without acceptance is byte-stable. Model-authored, so it never carries a setup command or timeout (<see cref="ModelAuthoredAcceptanceConverter"/>).</summary>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
[JsonConverter(typeof(ModelAuthoredAcceptanceConverter))]
public SupervisorAcceptanceSpec? Acceptance { get; init; }
}
Loading
Loading