From 951e1b8b10fab768461be2db6620ec276c2fe005 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 16:19:54 +0900 Subject: [PATCH 01/63] feat(g01): observe bounded idle drain ordering --- docs/evidence/g01-idle-drain.md | 80 +++ experiments/g01-scaleset/cmd/g01-live/main.go | 2 +- .../g01-scaleset/livecanary/approval.go | 4 +- .../livecanary/baseline_journal.go | 7 + experiments/g01-scaleset/livecanary/drain.go | 578 ++++++++++++++++++ .../g01-scaleset/livecanary/drain_driver.go | 184 ++++++ .../g01-scaleset/livecanary/drain_test.go | 341 +++++++++++ experiments/g01-scaleset/livecanary/driver.go | 34 +- .../g01-scaleset/livecanary/journal.go | 3 + .../livecanary/response_budget.go | 10 +- experiments/g01-scaleset/livecanary/sdk.go | 33 +- experiments/g02-auth/broker.go | 2 +- experiments/g02-auth/broker_admission_test.go | 2 +- experiments/g02-auth/broker_entry.go | 4 +- 14 files changed, 1261 insertions(+), 23 deletions(-) create mode 100644 docs/evidence/g01-idle-drain.md create mode 100644 experiments/g01-scaleset/livecanary/drain.go create mode 100644 experiments/g01-scaleset/livecanary/drain_driver.go create mode 100644 experiments/g01-scaleset/livecanary/drain_test.go diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md new file mode 100644 index 0000000..673ce7a --- /dev/null +++ b/docs/evidence/g01-idle-drain.md @@ -0,0 +1,80 @@ +# G01 idle-drain observation evidence + +Issue #71 adds an experiment-only observation phase. It does not close the +parent G01 goal, add a production provider, or authorize live execution. + +## Invariant and counterexample + +The invariant is: withdrawing listener capacity to zero may affect the next +poll, while the poll already admitted by the listener must retain the pinned +SDK's ACK-before-acquisition ordering. The counterexample is a response that +arrives before a request-written marker, an absent/ambiguous marker, or a +second poll that returns a message after withdrawal; each case is recorded as +inconclusive/quarantined and never treated as proof of a drain barrier. + +The transport hook records `WroteRequest` and holds the first response body +before the high-level listener parser receives it. `WroteRequest` is a +client-side transport fact only: `server_receipt` remains `unproven`, so the +synthetic test and any future live result cannot claim server acceptance or an +atomic server-side drain. The implementation calls the released listener's +public `SetMaxRunners(0)` callback and lets the listener perform its normal +ACK-before-acquire sequence. + +## Bounded observation + +The private journal record contains only fixed categories, the initial/zero +capacities, the poll and next-poll response categories, the seven nonnegative +scale-set counters (available, acquired, assigned, running, registered, busy, +idle), exact owned scale-set identity, exact runner identity when present, a +fixed ordering, sequence and timestamp. It excludes queue URLs, access +tokens, request/response bodies, JIT material and raw SDK errors. A drain event +marks work as observed during replay; an inconclusive event additionally +retains uncertainty. Neither outcome authorizes deletion, replay, +re-acquisition or runner removal. + +The prerequisite remains one already-idle, registered, owned canary runner +with zero available/acquired/assigned/running/busy work. No idle-worker +pre-provision seam was added; if a future live run needs one, it requires a +separate independent design review and explicit maintainer authorization. + +## TDD and verification + +Meaningful behavioral red was captured before the implementation existed: + +```text +cd experiments/g01-scaleset +go test ./livecanary -run 'TestDrainListener' -count=1 +``` + +The module failed to compile because the drain hook, listener runner, +boundary/category constants and experiment seam were undefined. After the +minimal implementation, the same behavior was green, followed by failure and +boundary coverage for missing request-written evidence, negative counters, +identity changes, duplicate ordering, response-before-write observations, +idle prerequisites, replay uncertainty and missing verification authority. + +Verified locally with the pinned `github.com/actions/scaleset v0.4.0` module: + +```text +cd experiments/g01-scaleset +go test ./livecanary -run 'TestDrain|TestCredentialAttestationMismatchAndExpiredTokenRejected' -count=1 +go test -race ./livecanary -run 'TestDrainListener' -count=1 +cd ../g02-auth +go test ./... -run 'TestBrokerFinitePhases|TestBrokerControllerApproval' -count=1 +``` + +All commands above passed. The listener test uses an `httptest` synthetic +transport and the real released listener; it is offline protocol evidence, +not live qualification. No credentials, runner/session/JIT operation, +workflow operation, app/keychain/launchd/Docker/Lima mutation or cleanup was +performed. + +## Remaining gate and rollback + +The live G01 gate remains unresolved until a separately authorized run uses an +immutable reviewed head, the approved private repository/workflow/resources, +one idle owned canary runner and independent review. A timing miss, missing +runner, stale zero, identity/statistics mismatch, unknown response, or +server-receipt ambiguity must remain inconclusive. Rollback is the focused +revert of the issue-71 commit/PR; no live cleanup or workflow replay is part +of rollback. diff --git a/experiments/g01-scaleset/cmd/g01-live/main.go b/experiments/g01-scaleset/cmd/g01-live/main.go index 80a7bf2..ef4284e 100644 --- a/experiments/g01-scaleset/cmd/g01-live/main.go +++ b/experiments/g01-scaleset/cmd/g01-live/main.go @@ -89,7 +89,7 @@ func runWithPreparation(args []string, in io.Reader, out io.Writer, revisionForB if *approvalPath != "" || *statePath != "" || *phase != "" || workerInputs || *pairedBinding != "" { return reject() } - fmt.Fprintln(out, "Controller-only phases: create, before-ack, after-ack, before-acquire, acquire-loss, jit-loss, inspect, cleanup. Paired terminal mode uses one same-process executable with explicit worker approval/state inputs and fixed terminal sequencing; no worker launch or workflow dispatch. Live execution requires an immutable reviewed build, exact private approval and controller-side broker input.") + fmt.Fprintln(out, "Controller-only phases: create, before-ack, after-ack, before-acquire, acquire-loss, jit-loss, drain, inspect, cleanup. Paired terminal mode uses one same-process executable with explicit worker approval/state inputs and fixed terminal sequencing; no worker launch or workflow dispatch. Live execution requires an immutable reviewed build, exact private approval and controller-side broker input.") return 0 } modeCount := 0 diff --git a/experiments/g01-scaleset/livecanary/approval.go b/experiments/g01-scaleset/livecanary/approval.go index b4adab2..30416db 100644 --- a/experiments/g01-scaleset/livecanary/approval.go +++ b/experiments/g01-scaleset/livecanary/approval.go @@ -12,7 +12,7 @@ var nonce = regexp.MustCompile(`^[a-f0-9]{32}$`) var sha = regexp.MustCompile(`^[a-f0-9]{40}$`) var actionsHost = regexp.MustCompile(`^[a-z0-9-]+(?:\.[a-z0-9-]+)*\.actions\.githubusercontent\.com$`) var workflowPath = regexp.MustCompile(`^\.github/workflows/[a-zA-Z0-9_-]+\.ya?ml$`) -var phases = []string{"create", "before-ack", "after-ack", "before-acquire", "acquire-loss", "jit-loss", "inspect", "cleanup"} +var phases = []string{"create", "before-ack", "after-ack", "before-acquire", "acquire-loss", "jit-loss", "drain", "inspect", "cleanup"} func (a Approval) setName() string { return "g01-" + a.OwnerNonce } func (a Approval) workerName() string { return a.setName() + "-worker-1" } @@ -24,7 +24,7 @@ func (a Approval) Validate(now time.Time) error { if a.Organization == ".." { return ErrApproval } - if !component.MatchString(a.Organization) || !component.MatchString(a.Repository) || !component.MatchString(a.Controller) || a.Organization == "." || a.Repository == "." || a.Repository == ".." || a.RepositoryID <= 0 || a.RunnerGroupID <= 0 || !nonce.MatchString(a.OwnerNonce) || !sha.MatchString(a.HarnessSHA) || !sha.MatchString(a.WorkflowSHA) || !workflowPath.MatchString(a.WorkflowPath) || !a.ExpiresAt.After(now) || a.ExpiresAt.After(now.Add(24*time.Hour)) || len(a.ActionsHosts) == 0 || len(a.ActionsHosts) > 8 || len(a.Phases) == 0 { + if !component.MatchString(a.Organization) || !component.MatchString(a.Repository) || !component.MatchString(a.Controller) || a.Organization == "." || a.Repository == "." || a.Repository == ".." || a.RepositoryID <= 0 || a.RunnerGroupID <= 0 || !nonce.MatchString(a.OwnerNonce) || !sha.MatchString(a.HarnessSHA) || !sha.MatchString(a.WorkflowSHA) || !workflowPath.MatchString(a.WorkflowPath) || !a.ExpiresAt.After(now) || a.ExpiresAt.After(now.Add(24*time.Hour)) || len(a.ActionsHosts) == 0 || len(a.ActionsHosts) > 8 || len(a.Phases) == 0 || len(a.Phases) > 9 { return ErrApproval } seen := map[string]bool{} diff --git a/experiments/g01-scaleset/livecanary/baseline_journal.go b/experiments/g01-scaleset/livecanary/baseline_journal.go index 371934c..ac7a956 100644 --- a/experiments/g01-scaleset/livecanary/baseline_journal.go +++ b/experiments/g01-scaleset/livecanary/baseline_journal.go @@ -116,6 +116,13 @@ func cloneEvent(e Event) Event { e.Authority = &x } e.RequestIDs = slices.Clone(e.RequestIDs) + if e.Drain != nil { + data, _ := json.Marshal(e.Drain) + var x drainObservation + _ = json.Unmarshal(data, &x) + x.Ordering = slices.Clone(e.Drain.Ordering) + e.Drain = &x + } if e.Baseline != nil { data, _ := json.Marshal(e.Baseline) var x baselineRecord diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go new file mode 100644 index 0000000..a55cb28 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -0,0 +1,578 @@ +package livecanary + +import ( + "context" + "errors" + "net/http" + "net/http/httptrace" + "net/url" + "slices" + "sync" + "time" + + "github.com/actions/scaleset" + "github.com/actions/scaleset/listener" +) + +const ( + drainObservationVersion = 1 + drainInitialCapacity = 1 + drainWithdrawnCapacity = 0 + + drainBoundaryRequestWritten = "request-written" + drainBoundaryResponseBeforeWrite = "response-before-request-written" + drainBoundaryUnresolved = "unresolved" + drainServerReceiptUnproven = "unproven" + drainMessagePresent = "present" + drainMessageAbsent = "absent" + drainMessageUnknown = "unknown" + drainResponseNotAttempted = "not-attempted" + drainResponseSucceeded = "succeeded" + drainResponseUnknown = "unknown" + drainOutcomeObserved = "observed" + drainOutcomeInconclusive = "inconclusive" +) + +var errDrainCollected = errors.New("drain observation collected") + +// drainStatistics is deliberately a closed, scalar schema. It contains no +// response text, queue URL, token or SDK object, and every value must be +// non-negative before it is persisted. +type drainStatistics struct { + Available int `json:"total_available_jobs"` + Acquired int `json:"total_acquired_jobs"` + Assigned int `json:"total_assigned_jobs"` + Running int `json:"total_running_jobs"` + Registered int `json:"total_registered_runners"` + Busy int `json:"total_busy_runners"` + Idle int `json:"total_idle_runners"` +} + +func newDrainStatistics(s *scaleset.RunnerScaleSetStatistic) (drainStatistics, error) { + if s == nil || s.TotalAvailableJobs < 0 || s.TotalAcquiredJobs < 0 || s.TotalAssignedJobs < 0 || s.TotalRunningJobs < 0 || s.TotalRegisteredRunners < 0 || s.TotalBusyRunners < 0 || s.TotalIdleRunners < 0 { + return drainStatistics{}, ErrQuarantine + } + return drainStatistics{Available: s.TotalAvailableJobs, Acquired: s.TotalAcquiredJobs, Assigned: s.TotalAssignedJobs, Running: s.TotalRunningJobs, Registered: s.TotalRegisteredRunners, Busy: s.TotalBusyRunners, Idle: s.TotalIdleRunners}, nil +} + +type drainPollObservation struct { + Capacity int `json:"capacity"` + Message string `json:"message"` + Statistics drainStatistics `json:"statistics"` + StatsKnown bool `json:"statistics_known"` + ACK string `json:"ack"` + Acquisition string `json:"acquisition"` +} + +type drainSetIdentity struct { + ID int `json:"id"` + Name string `json:"name"` + RunnerGroupID int `json:"runner_group_id"` + Label string `json:"label"` +} + +type drainRunnerIdentity struct { + ID int `json:"id"` + Name string `json:"name"` + ScaleSetID int `json:"scale_set_id"` +} + +type drainSnapshot struct { + Set drainSetIdentity `json:"set"` + Statistics drainStatistics `json:"statistics"` + Runner *drainRunnerIdentity `json:"runner,omitempty"` +} + +// drainObservation is private journal evidence. Its JSON shape is bounded and +// contains only fixed categories, identities and non-negative counters. +type drainObservation struct { + Version int `json:"version"` + Outcome string `json:"outcome"` + InitialCapacity int `json:"initial_capacity"` + WithdrawnCapacity int `json:"withdrawn_capacity"` + Boundary string `json:"boundary"` + ServerReceipt string `json:"server_receipt"` + ResponseHeld bool `json:"response_held"` + Poll drainPollObservation `json:"poll"` + NextPoll drainPollObservation `json:"next_poll"` + Before drainSnapshot `json:"before"` + After drainSnapshot `json:"after"` + Ordering []string `json:"ordering"` + Sequence int `json:"sequence"` + ObservedAt time.Time `json:"observed_at"` +} + +func validDrainResponse(value string) bool { + return value == drainResponseNotAttempted || value == drainResponseSucceeded || value == drainResponseUnknown +} + +func validDrainPoll(p drainPollObservation, wantCapacity int) bool { + if p.Capacity != wantCapacity || p.Message != drainMessagePresent && p.Message != drainMessageAbsent && p.Message != drainMessageUnknown || !validDrainResponse(p.ACK) || !validDrainResponse(p.Acquisition) { + return false + } + if p.Statistics.Available < 0 || p.Statistics.Acquired < 0 || p.Statistics.Assigned < 0 || p.Statistics.Running < 0 || p.Statistics.Registered < 0 || p.Statistics.Busy < 0 || p.Statistics.Idle < 0 { + return false + } + return true +} + +func validDrainSnapshot(s drainSnapshot, expected drainSetIdentity) bool { + if s.Set != expected || s.Set.ID <= 0 || s.Set.RunnerGroupID <= 0 || s.Set.Name == "" || s.Set.Label == "" || !baselineText(s.Set.Name, 128) || !baselineText(s.Set.Label, 128) { + return false + } + if s.Statistics.Available < 0 || s.Statistics.Acquired < 0 || s.Statistics.Assigned < 0 || s.Statistics.Running < 0 || s.Statistics.Registered < 0 || s.Statistics.Busy < 0 || s.Statistics.Idle < 0 { + return false + } + if s.Runner != nil && (s.Runner.ID <= 0 || s.Runner.ScaleSetID != s.Set.ID || s.Runner.Name == "" || !baselineText(s.Runner.Name, 256)) { + return false + } + return true +} + +func validDrainObservation(o *drainObservation) bool { + if o == nil || o.Version != drainObservationVersion || (o.Outcome != drainOutcomeObserved && o.Outcome != drainOutcomeInconclusive) || o.InitialCapacity != drainInitialCapacity || o.WithdrawnCapacity != drainWithdrawnCapacity || o.ServerReceipt != drainServerReceiptUnproven || o.Sequence <= 0 || o.ObservedAt.IsZero() || len(o.Ordering) > 8 { + return false + } + if o.Boundary != drainBoundaryRequestWritten && o.Boundary != drainBoundaryResponseBeforeWrite && o.Boundary != drainBoundaryUnresolved || !validDrainPoll(o.Poll, drainInitialCapacity) || !validDrainPoll(o.NextPoll, drainWithdrawnCapacity) || !validDrainSnapshot(o.Before, o.Before.Set) || !validDrainSnapshot(o.After, o.Before.Set) { + return false + } + if o.Outcome == drainOutcomeObserved && (o.Boundary != drainBoundaryRequestWritten || !o.ResponseHeld) { + return false + } + seen := map[string]bool{} + for _, item := range o.Ordering { + if item != "poll-old" && item != "ack" && item != "acquire" && item != "poll-zero" || seen[item] { + return false + } + seen[item] = true + } + if o.Outcome == drainOutcomeObserved { + if o.NextPoll.Message != drainMessageAbsent || o.NextPoll.ACK != drainResponseNotAttempted || o.NextPoll.Acquisition != drainResponseNotAttempted { + return false + } + if o.Poll.Message == drainMessagePresent { + if o.Poll.ACK != drainResponseSucceeded || (o.Poll.Acquisition != drainResponseNotAttempted && o.Poll.Acquisition != drainResponseSucceeded) { + return false + } + } else if o.Poll.Message == drainMessageAbsent && (o.Poll.ACK != drainResponseNotAttempted || o.Poll.Acquisition != drainResponseNotAttempted) { + return false + } else if o.Poll.Message != drainMessageAbsent { + return false + } + want := []string{"poll-old"} + if o.Poll.Message == drainMessagePresent { + want = append(want, "ack") + if o.Poll.Acquisition == drainResponseSucceeded { + want = append(want, "acquire") + } + } + want = append(want, "poll-zero") + if !slices.Equal(o.Ordering, want) { + return false + } + } + return true +} + +func (o *drainObservation) poll(index int) *drainPollObservation { + if index == 1 { + return &o.Poll + } + return &o.NextPoll +} + +// drainPollHook is installed only in the experiment's SDK client. WroteRequest +// is a client-side transport fact; it does not prove GitHub accepted the +// request. The first response body is held before it reaches the SDK parser so +// the listener's capacity transition and ACK/acquisition order remain visible. +type drainPollHook struct { + inner http.RoundTripper + target string + wrote chan struct{} + response chan struct{} + release chan struct{} + + mu sync.Mutex + wroteRequest bool + responseBeforeWrite bool + responseHeld bool + invalid bool + pollAttempts int + onRequestWritten func() + wroteOnce sync.Once + responseOnce sync.Once + releaseOnce sync.Once +} + +func newDrainPollHook(target string) *drainPollHook { + return &drainPollHook{inner: http.DefaultTransport, target: target, wrote: make(chan struct{}), response: make(chan struct{}), release: make(chan struct{})} +} + +func (h *drainPollHook) matches(req *http.Request) bool { + if h == nil || req == nil || req.Method != http.MethodGet || req.URL == nil { + return false + } + h.mu.Lock() + target := h.target + h.mu.Unlock() + want, err := url.Parse(target) + if err != nil || want.Scheme == "" || want.Host == "" { + return false + } + // Queue access tokens may be embedded in the URL. Compare only the + // transport destination/path; never copy the query into an observation. + return req.URL.Scheme == want.Scheme && req.URL.Host == want.Host && req.URL.Path == want.Path +} + +func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { + if h == nil || h.inner == nil || !h.matches(req) { + if h == nil || h.inner == nil { + return nil, ErrQuarantine + } + return h.inner.RoundTrip(req) + } + h.mu.Lock() + h.pollAttempts++ + first := h.pollAttempts == 1 + h.mu.Unlock() + if !first { + return h.inner.RoundTrip(req) + } + + trace := &httptrace.ClientTrace{WroteRequest: func(info httptrace.WroteRequestInfo) { + h.mu.Lock() + if info.Err != nil { + h.invalid = true + } else { + h.wroteRequest = true + } + h.mu.Unlock() + if info.Err == nil { + h.wroteOnce.Do(func() { + if h.onRequestWritten != nil { + h.onRequestWritten() + } + close(h.wrote) + }) + } else { + h.wroteOnce.Do(func() { close(h.wrote) }) + } + }} + response, err := h.inner.RoundTrip(req.WithContext(httptrace.WithClientTrace(req.Context(), trace))) + if err != nil { + h.wroteOnce.Do(func() { close(h.wrote) }) + h.responseOnce.Do(func() { close(h.response) }) + return nil, err + } + if response == nil { + h.mu.Lock() + h.invalid = true + h.mu.Unlock() + h.wroteOnce.Do(func() { close(h.wrote) }) + h.responseOnce.Do(func() { close(h.response) }) + return nil, ErrQuarantine + } + h.mu.Lock() + if !h.wroteRequest { + h.responseBeforeWrite = true + } + h.responseHeld = true + h.mu.Unlock() + if response.Body == nil { + response.Body = http.NoBody + } + response.Body = &drainHeldBody{source: response.Body, release: h.release} + h.responseOnce.Do(func() { close(h.response) }) + return response, nil +} + +func (h *drainPollHook) releaseResponse() { + if h == nil { + return + } + h.releaseOnce.Do(func() { close(h.release) }) +} + +func (h *drainPollHook) boundary() (string, bool, bool) { + h.mu.Lock() + defer h.mu.Unlock() + switch { + case h.wroteRequest && h.responseHeld && !h.responseBeforeWrite: + return drainBoundaryRequestWritten, true, !h.invalid + case h.responseBeforeWrite: + return drainBoundaryResponseBeforeWrite, h.responseHeld, false + default: + return drainBoundaryUnresolved, h.responseHeld, false + } +} + +type drainHeldBody struct { + source ioReadCloser + release <-chan struct{} +} + +type ioReadCloser interface { + Read([]byte) (int, error) + Close() error +} + +func (b *drainHeldBody) Read(p []byte) (int, error) { + <-b.release + return b.source.Read(p) +} +func (b *drainHeldBody) Close() error { return b.source.Close() } + +// drainClient records only high-level listener effects and the two bounded +// polls. It never changes ACK or acquisition order. +type drainClient struct { + inner listener.Client + validate func(context.Context, *scaleset.RunnerScaleSetMessage) error + obs *drainObservation + mu sync.Mutex + polls int + messageID int + requestID int64 +} + +func (c *drainClient) Session() scaleset.RunnerScaleSetSession { return c.inner.Session() } + +func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scaleset.RunnerScaleSetMessage, error) { + c.mu.Lock() + if capacity != drainInitialCapacity && capacity != drainWithdrawnCapacity || c.polls >= 2 { + c.mu.Unlock() + return nil, ErrQuarantine + } + c.polls++ + index := c.polls + c.obs.poll(index).Capacity = capacity + c.obs.poll(index).ACK = drainResponseNotAttempted + c.obs.poll(index).Acquisition = drainResponseNotAttempted + c.obs.Ordering = append(c.obs.Ordering, map[int]string{1: "poll-old", 2: "poll-zero"}[index]) + c.mu.Unlock() + + message, err := c.inner.GetMessage(ctx, last, capacity) + if err != nil { + c.mu.Lock() + c.obs.poll(index).Message = drainMessageUnknown + c.mu.Unlock() + return nil, err + } + if message == nil { + c.mu.Lock() + c.obs.poll(index).Message = drainMessageAbsent + c.mu.Unlock() + return nil, nil + } + stats, statsErr := newDrainStatistics(message.Statistics) + if statsErr != nil || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) > 1 || len(message.JobAvailableMessages) == 1 && (message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0) { + c.mu.Lock() + c.obs.poll(index).Message = drainMessageUnknown + c.mu.Unlock() + return nil, ErrQuarantine + } + if index == 2 { + // A message after withdrawal is retained as an unresolved race; do not + // ACK or acquire a second message in this bounded phase. + c.mu.Lock() + c.obs.poll(index).Message = drainMessageUnknown + c.mu.Unlock() + return nil, ErrQuarantine + } + if c.validate != nil { + if err := c.validate(ctx, message); err != nil { + c.mu.Lock() + c.obs.poll(index).Message = drainMessageUnknown + c.mu.Unlock() + return nil, err + } + } + c.mu.Lock() + c.obs.poll(index).Message = drainMessagePresent + c.obs.poll(index).Statistics = stats + c.obs.poll(index).StatsKnown = true + c.messageID = message.MessageID + if len(message.JobAvailableMessages) == 1 { + c.requestID = message.JobAvailableMessages[0].RunnerRequestID + } + c.mu.Unlock() + return message, nil +} + +func (c *drainClient) DeleteMessage(ctx context.Context, id int) error { + err := c.inner.DeleteMessage(ctx, id) + c.mu.Lock() + defer c.mu.Unlock() + index := c.polls + if index < 1 || index > 2 || c.messageID <= 0 || id != c.messageID { + return ErrQuarantine + } + c.obs.poll(index).ACK = drainResponseSucceeded + c.obs.Ordering = append(c.obs.Ordering, "ack") + if err != nil { + c.obs.poll(index).ACK = drainResponseUnknown + } + return err +} + +func (c *drainClient) AcquireJobs(ctx context.Context, ids []int64) ([]int64, error) { + if len(ids) != 1 { + return nil, ErrQuarantine + } + got, err := c.inner.AcquireJobs(ctx, slices.Clone(ids)) + c.mu.Lock() + defer c.mu.Unlock() + index := c.polls + if index < 1 || index > 2 || c.requestID <= 0 || ids[0] != c.requestID { + return nil, ErrQuarantine + } + c.obs.poll(index).Acquisition = drainResponseSucceeded + c.obs.Ordering = append(c.obs.Ordering, "acquire") + if err != nil { + c.obs.poll(index).Acquisition = drainResponseUnknown + return nil, err + } + if !slices.Equal(got, ids) { + c.obs.poll(index).Acquisition = drainResponseUnknown + return nil, ErrQuarantine + } + return got, nil +} + +type drainScaler struct{ client *drainClient } + +func (s drainScaler) HandleDesiredRunnerCount(_ context.Context, count int) (int, error) { + if count < 0 { + return 0, ErrQuarantine + } + s.client.mu.Lock() + polls := s.client.polls + s.client.mu.Unlock() + if polls >= 2 { + return 0, errDrainCollected + } + return min(count, drainInitialCapacity), nil +} +func (drainScaler) HandleJobStarted(context.Context, *scaleset.JobStarted) error { + return ErrQuarantine +} +func (drainScaler) HandleJobCompleted(context.Context, *scaleset.JobCompleted) error { + return ErrQuarantine +} + +func runDrainListener(ctx context.Context, client listener.Client, setID int, hook *drainPollHook) (drainObservation, error) { + obs := drainObservation{Version: drainObservationVersion, Outcome: drainOutcomeInconclusive, InitialCapacity: drainInitialCapacity, WithdrawnCapacity: drainWithdrawnCapacity, Boundary: drainBoundaryUnresolved, ServerReceipt: drainServerReceiptUnproven, Sequence: 1, ObservedAt: time.Now().UTC()} + obs.Poll = drainPollObservation{Capacity: drainInitialCapacity, Message: drainMessageUnknown, ACK: drainResponseNotAttempted, Acquisition: drainResponseNotAttempted} + obs.NextPoll = drainPollObservation{Capacity: drainWithdrawnCapacity, Message: drainMessageUnknown, ACK: drainResponseNotAttempted, Acquisition: drainResponseNotAttempted} + if ctx == nil || client == nil || setID <= 0 || hook == nil { + return obs, ErrApproval + } + hook.mu.Lock() + target := hook.target + hook.mu.Unlock() + if target == "" { + return obs, ErrApproval + } + initial := client.Session() + if initial.SessionID == [16]byte{} || initial.Statistics == nil { + return obs, ErrQuarantine + } + if stats, err := newDrainStatistics(initial.Statistics); err != nil { + return obs, err + } else { + obs.Poll.Statistics = stats + obs.Poll.StatsKnown = true + } + c := &drainClient{inner: client, obs: &obs} + l, err := listener.New(c, listener.Config{ScaleSetID: setID, MaxRunners: drainInitialCapacity}) + if err != nil { + return obs, ErrQuarantine + } + hook.onRequestWritten = func() { l.SetMaxRunners(drainWithdrawnCapacity) } + runResult := make(chan error, 1) + go func() { runResult <- l.Run(ctx, drainScaler{client: c}) }() + + release := false + defer func() { + if !release { + hook.releaseResponse() + } + }() + responseCaptured := false + select { + case <-hook.wrote: + // The callback has already called the public listener method. This is + // deliberately a client transport boundary, not server acceptance. + case <-hook.response: + // A response before the request-written marker is a timing miss. Release + // the held body only so the listener can finish classifying the result; + // never promote this path to an observed in-flight boundary. If both + // channels are ready, retain a valid request-written boundary instead of + // letting select's choice turn a valid run into a flaky skip. + boundary, held, proven := hook.boundary() + obs.Boundary, obs.ResponseHeld = boundary, held + hook.releaseResponse() + release = true + if !proven { + select { + case err := <-runResult: + if errors.Is(err, errDrainCollected) { + obs.Outcome = drainOutcomeInconclusive + return obs, ErrNoMessage + } + return obs, drainBoundaryError(&obs, hook, err) + case <-ctx.Done(): + return obs, ErrQuarantine + } + } + responseCaptured = true + case err := <-runResult: + return obs, drainBoundaryError(&obs, hook, err) + case <-ctx.Done(): + return obs, ErrQuarantine + } + if !responseCaptured { + select { + case <-hook.response: + boundary, held, proven := hook.boundary() + obs.Boundary, obs.ResponseHeld = boundary, held + if !proven { + obs.Outcome = drainOutcomeInconclusive + } + hook.releaseResponse() + release = true + case err := <-runResult: + return obs, drainBoundaryError(&obs, hook, err) + case <-ctx.Done(): + return obs, ErrQuarantine + } + } + err = <-runResult + if errors.Is(err, errDrainCollected) { + if obs.Boundary == "" { + obs.Boundary, obs.ResponseHeld, _ = hook.boundary() + } + if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld { + obs.Outcome = drainOutcomeObserved + return obs, nil + } + obs.Outcome = drainOutcomeInconclusive + return obs, ErrNoMessage + } + return obs, drainBoundaryError(&obs, hook, err) +} + +func drainBoundaryError(obs *drainObservation, hook *drainPollHook, err error) error { + if obs != nil && hook != nil { + obs.Boundary, obs.ResponseHeld, _ = hook.boundary() + } + if err == nil { + return ErrNoMessage + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return ErrQuarantine + } + if obs != nil && obs.Boundary == drainBoundaryUnresolved { + return ErrNoMessage + } + return ErrQuarantine +} diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go new file mode 100644 index 0000000..dda69d4 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -0,0 +1,184 @@ +package livecanary + +import ( + "context" + "errors" + "slices" + + "github.com/actions/scaleset" +) + +type drainSessionOpener interface { + OpenDrainSession(context.Context, int, string, *drainPollHook) (Session, error) +} + +type journaledDrainClient struct { + d *Driver + inner Session + sessionID string +} + +func (c *journaledDrainClient) Session() scaleset.RunnerScaleSetSession { return c.inner.Session() } + +func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity int) (*scaleset.RunnerScaleSetMessage, error) { + var message *scaleset.RunnerScaleSetMessage + err := c.d.effect(ctx, "observe-poll", nil, func(call context.Context) (Event, error) { + var err error + message, err = c.inner.GetMessage(call, last, capacity) + if err != nil { + return Event{}, err + } + if message == nil { + return Event{SessionID: c.sessionID}, nil + } + if message.MessageID <= 0 || len(message.JobAvailableMessages) > 1 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || message.Statistics == nil { + return Event{}, ErrRemote + } + for _, job := range message.JobAvailableMessages { + if job == nil || job.RunnerRequestID <= 0 || job.WorkflowRunID != c.d.Approval.WorkflowRunID || job.OwnerName != c.d.Approval.Organization || job.RepositoryName != c.d.Approval.Repository { + return Event{}, ErrApproval + } + if _, err := boundedRead(call, func(verify context.Context) (bool, error) { + return true, c.d.API.VerifyRun(verify, c.d.Approval, job.WorkflowRunID) + }); err != nil { + return Event{}, ErrApproval + } + } + return Event{ID: message.MessageID, SessionID: c.sessionID}, nil + }) + if err != nil { + return nil, err + } + return message, nil +} + +func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error { + return c.d.effect(ctx, "ack", nil, func(call context.Context) (Event, error) { + if err := c.inner.DeleteMessage(call, id); err != nil { + return Event{}, err + } + return Event{ID: id, SessionID: c.sessionID}, nil + }) +} + +func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([]int64, error) { + ids = slices.Clone(ids) + var got []int64 + err := c.d.effect(ctx, "acquire", ids, func(call context.Context) (Event, error) { + var err error + got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) + if err != nil || !slices.Equal(got, ids) { + return Event{}, errors.New("acquisition response did not match the one-shot request") + } + return Event{RequestIDs: slices.Clone(ids), SessionID: c.sessionID}, nil + }) + if err != nil { + return nil, err + } + return slices.Clone(got), nil +} + +func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (drainSnapshot, error) { + var snapshot drainSnapshot + var set *scaleset.RunnerScaleSet + if err := d.effect(ctx, "observe-owned", nil, func(call context.Context) (Event, error) { + var err error + set, err = d.API.GetScaleSet(call, setID) + if err != nil || set == nil || set.ID != setID || set.Name != d.Approval.setName() || set.RunnerGroupID != d.Approval.RunnerGroupID || !set.RunnerSetting.DisableUpdate { + return Event{}, ErrRemote + } + label := "" + for _, candidate := range set.Labels { + if candidate.Name == d.Approval.setName() { + label = candidate.Name + } + } + stats, err := newDrainStatistics(set.Statistics) + if err != nil || label == "" { + return Event{}, ErrRemote + } + snapshot.Set = drainSetIdentity{ID: set.ID, Name: set.Name, RunnerGroupID: set.RunnerGroupID, Label: label} + snapshot.Statistics = stats + return Event{ID: set.ID}, nil + }); err != nil { + return drainSnapshot{}, err + } + var runner *scaleset.RunnerReference + if err := d.effect(ctx, "observe-runner", nil, func(call context.Context) (Event, error) { + var err error + runner, err = d.API.FindRunner(call, d.Approval.workerName()) + if err != nil { + return Event{}, err + } + if runner == nil { + return Event{}, nil + } + if runner.ID <= 0 || runner.Name != d.Approval.workerName() || runner.RunnerScaleSetID != setID { + return Event{Work: workUnresolved}, nil + } + snapshot.Runner = &drainRunnerIdentity{ID: runner.ID, Name: runner.Name, ScaleSetID: runner.RunnerScaleSetID} + return Event{ID: runner.ID}, nil + }); err != nil { + return drainSnapshot{}, err + } + _ = stage // Stage is retained by the caller's before/after final payload. + return snapshot, nil +} + +func validDrainIdlePrerequisite(s drainSnapshot) bool { + return s.Runner != nil && s.Statistics.Available == 0 && s.Statistics.Acquired == 0 && s.Statistics.Assigned == 0 && s.Statistics.Running == 0 && s.Statistics.Registered == 1 && s.Statistics.Busy == 0 && s.Statistics.Idle == 1 +} + +func (d *Driver) drain(ctx context.Context, setID int) error { + opener, ok := d.API.(drainSessionOpener) + if !ok { + return ErrApproval + } + before, err := d.drainSnapshot(ctx, setID, "before") + if err != nil || !validDrainIdlePrerequisite(before) { + return ErrQuarantine + } + hook := newDrainPollHook("") + var session Session + var sessionID string + err = d.effect(ctx, "session-open", nil, func(call context.Context) (Event, error) { + var err error + session, err = opener.OpenDrainSession(call, setID, d.Approval.setName(), hook) + if err != nil || session == nil { + return Event{}, ErrRemote + } + current := session.Session() + if current.SessionID == [16]byte{} || current.OwnerName != d.Approval.setName() || current.Statistics == nil { + return Event{}, ErrQuarantine + } + stats, err := newDrainStatistics(current.Statistics) + if err != nil || stats != before.Statistics { + return Event{}, ErrQuarantine + } + sessionID = current.SessionID.String() + return Event{SessionID: sessionID}, nil + }) + if err != nil { + return err + } + journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID} + obs, runErr := runDrainListener(ctx, journaled, setID, hook) + closeErr := d.effect(ctx, "session-close", nil, func(call context.Context) (Event, error) { + if err := session.Close(call); err != nil { + return Event{}, err + } + return Event{SessionID: sessionID}, nil + }) + if closeErr != nil { + return closeErr + } + after, afterErr := d.drainSnapshot(ctx, setID, "after") + if afterErr != nil { + return afterErr + } + obs.Before, obs.After = before, after + if err := d.record(Event{Kind: "observation", Operation: "drain", Drain: &obs}); err != nil { + return err + } + return runErr +} diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go new file mode 100644 index 0000000..08e9248 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -0,0 +1,341 @@ +package livecanary + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/actions/scaleset" + "github.com/google/uuid" +) + +func TestDrainListenerWithdrawsWhilePollResponseIsHeld(t *testing.T) { + var mu sync.Mutex + var capacities []string + var order []string + polls := 0 + message := map[string]any{ + "messageId": 7, + "messageType": "RunnerScaleSetJobMessages", + "statistics": map[string]int{ + "totalAvailableJobs": 1, + "totalAcquiredJobs": 0, + "totalAssignedJobs": 1, + "totalRunningJobs": 0, + "totalRegisteredRunners": 1, + "totalBusyRunners": 0, + "totalIdleRunners": 1, + }, + "body": "[]", + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + if r.Method == http.MethodGet { + capacities = append(capacities, r.Header.Get(scaleset.HeaderScaleSetMaxCapacity)) + polls++ + if polls == 1 { + data, _ := json.Marshal(message) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = w.Write(data) + mu.Unlock() + return + } + w.WriteHeader(http.StatusAccepted) + mu.Unlock() + return + } + if r.Method == http.MethodDelete { + order = append(order, "ack") + w.WriteHeader(http.StatusNoContent) + mu.Unlock() + return + } + if r.Method == http.MethodPost { + order = append(order, "acquire") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, `{"count":1,"value":[11]}`) + mu.Unlock() + return + } + mu.Unlock() + http.NotFound(w, r) + })) + defer server.Close() + + hook := newDrainPollHook(server.URL) + client := &http.Client{Transport: hook} + session := &drainSyntheticSession{ + client: client, + initial: scaleset.RunnerScaleSetSession{ + SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: server.URL, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, + order: &order, + } + + observation, err := runDrainListener(context.Background(), session, 7, hook) + if err != nil { + t.Fatalf("drain listener: %v", err) + } + mu.Lock() + gotCapacities := append([]string(nil), capacities...) + gotOrder := append([]string(nil), order...) + mu.Unlock() + if len(gotCapacities) != 2 || gotCapacities[0] != "1" || gotCapacities[1] != "0" { + t.Fatalf("poll capacities = %v, want [1 0]", gotCapacities) + } + if len(gotOrder) != 2 || gotOrder[0] != "ack" || gotOrder[1] != "acquire" { + t.Fatalf("side-effect order = %v, want ACK then acquire", gotOrder) + } + if observation.Boundary != drainBoundaryRequestWritten || !observation.ResponseHeld || observation.ServerReceipt != drainServerReceiptUnproven { + t.Fatalf("boundary = %+v, want request-written/held/server-unproven", observation) + } + if observation.Poll.ACK != drainResponseSucceeded || observation.Poll.Acquisition != drainResponseSucceeded || observation.NextPoll.Capacity != 0 { + t.Fatalf("observation = %+v", observation) + } +} + +func TestDrainListenerRejectsMissingRequestWrittenBoundary(t *testing.T) { + hook := newDrainPollHook("http://fixture.invalid/queue") + session := &drainSyntheticSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: "http://fixture.invalid/queue", Statistics: &scaleset.RunnerScaleSetStatistic{}}} + if _, err := runDrainListener(context.Background(), session, 7, hook); !errors.Is(err, ErrNoMessage) { + t.Fatalf("missing boundary error = %v, want unresolved", err) + } +} + +func TestDrainListenerMarksResponseBeforeWriteInconclusive(t *testing.T) { + hook := newDrainPollHook("http://fixture.invalid/queue") + hook.inner = &drainNoTraceTransport{} + session := &drainSyntheticSession{ + client: &http.Client{Transport: hook}, + initial: scaleset.RunnerScaleSetSession{ + SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: "http://fixture.invalid/queue", + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, + order: new([]string), + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + observation, err := runDrainListener(ctx, session, 7, hook) + if !errors.Is(err, ErrNoMessage) || observation.Boundary != drainBoundaryResponseBeforeWrite || observation.Outcome != drainOutcomeInconclusive { + t.Fatalf("response-before-write = %+v, err=%v; want inconclusive", observation, err) + } +} + +func TestDrainObservationIsBoundedAndFailClosed(t *testing.T) { + observation := validDrainTestObservation() + if !validDrainObservation(&observation) { + t.Fatal("valid drain observation rejected") + } + + for name, mutate := range map[string]func(*drainObservation){ + "negative-counter": func(o *drainObservation) { o.Poll.Statistics.Available = -1 }, + "negative-unknown-counter": func(o *drainObservation) { o.NextPoll.Statistics.Idle = -1; o.NextPoll.StatsKnown = false }, + "identity-change": func(o *drainObservation) { o.After.Set.ID++ }, + "duplicate-order": func(o *drainObservation) { o.Ordering = append(o.Ordering, "ack") }, + "response-before-write-observed": func(o *drainObservation) { + o.Boundary = drainBoundaryResponseBeforeWrite + }, + "server-receipt-claim": func(o *drainObservation) { o.ServerReceipt = "accepted" }, + } { + t.Run(name, func(t *testing.T) { + candidate := observation + candidate.Ordering = append([]string(nil), observation.Ordering...) + mutate(&candidate) + if validDrainObservation(&candidate) { + t.Fatal("invalid drain observation accepted") + } + }) + } + + raw, err := json.Marshal(observation) + if err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"token", "authorization", "queue-url", "raw-error", "jit"} { + if strings.Contains(strings.ToLower(string(raw)), forbidden) { + t.Fatalf("bounded observation contains %q: %s", forbidden, raw) + } + } + + state := replay([]Event{{Kind: "observation", Operation: "drain", Drain: &observation}}) + if !state.workObserved || state.uncertain { + t.Fatalf("observed drain replay = %+v, want work retained without uncertainty", state) + } + observation.Outcome = drainOutcomeInconclusive + state = replay([]Event{{Kind: "observation", Operation: "drain", Drain: &observation}}) + if !state.workObserved || !state.uncertain { + t.Fatalf("inconclusive drain replay = %+v, want retained and quarantined", state) + } +} + +func TestDrainIdlePrerequisiteRejectsAmbiguousState(t *testing.T) { + observation := validDrainTestObservation() + before := observation.Before + if !validDrainIdlePrerequisite(before) { + t.Fatal("valid idle prerequisite rejected") + } + for name, mutate := range map[string]func(*drainSnapshot){ + "missing-runner": func(s *drainSnapshot) { s.Runner = nil }, + "busy": func(s *drainSnapshot) { s.Statistics.Busy = 1 }, + "assigned": func(s *drainSnapshot) { s.Statistics.Assigned = 1 }, + "two-runners": func(s *drainSnapshot) { s.Statistics.Registered = 2 }, + "zero-idle": func(s *drainSnapshot) { s.Statistics.Idle = 0 }, + } { + t.Run(name, func(t *testing.T) { + candidate := before + if before.Runner != nil { + runner := *before.Runner + candidate.Runner = &runner + } + mutate(&candidate) + if validDrainIdlePrerequisite(candidate) { + t.Fatal("ambiguous idle prerequisite accepted") + } + }) + } +} + +func TestDrainRequiresVerificationAuthority(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + c := credentials(a) + c.VerificationToken = "" + if _, err := NewSDKAPI(a, c); !errors.Is(err, ErrApproval) { + t.Fatalf("drain without verification authority = %v, want approval rejection", err) + } +} + +func validDrainTestObservation() drainObservation { + set := drainSetIdentity{ID: 7, Name: "g01-test-set", RunnerGroupID: 3, Label: "g01-test-set"} + runner := &drainRunnerIdentity{ID: 19, Name: "g01-test-worker-1", ScaleSetID: 7} + beforeStats := drainStatistics{Registered: 1, Idle: 1} + return drainObservation{ + Version: drainObservationVersion, Outcome: drainOutcomeObserved, + InitialCapacity: drainInitialCapacity, WithdrawnCapacity: drainWithdrawnCapacity, + Boundary: drainBoundaryRequestWritten, ServerReceipt: drainServerReceiptUnproven, + ResponseHeld: true, + Poll: drainPollObservation{Capacity: drainInitialCapacity, Message: drainMessagePresent, Statistics: drainStatistics{Available: 1, Assigned: 1, Registered: 1, Idle: 1}, StatsKnown: true, ACK: drainResponseSucceeded, Acquisition: drainResponseSucceeded}, + NextPoll: drainPollObservation{Capacity: drainWithdrawnCapacity, Message: drainMessageAbsent, Statistics: beforeStats, StatsKnown: true, ACK: drainResponseNotAttempted, Acquisition: drainResponseNotAttempted}, + Before: drainSnapshot{Set: set, Statistics: beforeStats, Runner: runner}, + After: drainSnapshot{Set: set, Statistics: beforeStats, Runner: runner}, + Ordering: []string{"poll-old", "ack", "acquire", "poll-zero"}, Sequence: 1, ObservedAt: time.Unix(1, 0).UTC(), + } +} + +type drainNoTraceTransport struct { + gets int +} + +func (t *drainNoTraceTransport) RoundTrip(req *http.Request) (*http.Response, error) { + response := &http.Response{Header: make(http.Header), Request: req} + switch req.Method { + case http.MethodGet: + t.gets++ + if t.gets == 1 { + response.StatusCode = http.StatusOK + response.Status = http.StatusText(http.StatusOK) + response.Body = io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalRegisteredRunners":1,"totalIdleRunners":1},"body":"[]"}`)) + } else { + response.StatusCode = http.StatusAccepted + response.Status = http.StatusText(http.StatusAccepted) + response.Body = http.NoBody + } + case http.MethodDelete: + response.StatusCode = http.StatusNoContent + response.Status = http.StatusText(http.StatusNoContent) + response.Body = http.NoBody + case http.MethodPost: + response.StatusCode = http.StatusOK + response.Status = http.StatusText(http.StatusOK) + response.Body = io.NopCloser(strings.NewReader(`{"count":1,"value":[11]}`)) + default: + response.StatusCode = http.StatusNotFound + response.Status = http.StatusText(http.StatusNotFound) + response.Body = http.NoBody + } + return response, nil +} + +type drainSyntheticSession struct { + client *http.Client + initial scaleset.RunnerScaleSetSession + order *[]string +} + +func (s *drainSyntheticSession) Session() scaleset.RunnerScaleSetSession { return s.initial } +func (s *drainSyntheticSession) Close(context.Context) error { return nil } +func (s *drainSyntheticSession) GetMessage(ctx context.Context, _, capacity int) (*scaleset.RunnerScaleSetMessage, error) { + if s.client == nil { + return nil, errors.New("synthetic transport unavailable") + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.initial.MessageQueueURL, nil) + if err != nil { + return nil, err + } + req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(capacity)) + resp, err := s.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode == http.StatusAccepted { + return nil, nil + } + if resp.StatusCode != http.StatusOK { + return nil, errors.New("synthetic poll failed") + } + if _, err := io.ReadAll(resp.Body); err != nil { + return nil, err + } + return &scaleset.RunnerScaleSetMessage{ + MessageID: 7, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalAvailableJobs: 1, TotalAssignedJobs: 1, TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + JobAvailableMessages: []*scaleset.JobAvailable{{JobMessageBase: scaleset.JobMessageBase{RunnerRequestID: 11}}}, + }, nil +} +func (s *drainSyntheticSession) DeleteMessage(ctx context.Context, id int) error { + if s.order == nil { + return errors.New("missing order sink") + } + if err := appendSyntheticMethod(ctx, s.client, http.MethodDelete, s.initial.MessageQueueURL+"/"+strconv.Itoa(id), s.order); err != nil { + return err + } + return nil +} +func (s *drainSyntheticSession) AcquireJobs(ctx context.Context, ids []int64) ([]int64, error) { + if len(ids) != 1 { + return nil, errors.New("unexpected request count") + } + if err := appendSyntheticMethod(ctx, s.client, http.MethodPost, s.initial.MessageQueueURL+"/acquirejobs", s.order); err != nil { + return nil, err + } + return append([]int64(nil), ids...), nil +} + +func appendSyntheticMethod(ctx context.Context, client *http.Client, method, target string, order *[]string) error { + req, err := http.NewRequestWithContext(ctx, method, target, nil) + if err != nil { + return err + } + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return errors.New("synthetic side effect failed") + } + return nil +} diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index f273b69..e02b1d1 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -41,18 +41,19 @@ type Approval struct { } type Event struct { - Baseline *baselineRecord `json:"baseline,omitempty"` - Authority *phaseAuthority `json:"authority,omitempty"` - Sequence int `json:"sequence"` - Kind string `json:"kind"` - Operation string `json:"operation,omitempty"` - ID int `json:"id,omitempty"` - SessionID string `json:"session_id,omitempty"` - RequestIDs []int64 `json:"request_ids,omitempty"` - Count int `json:"count,omitempty"` - Digest string `json:"digest,omitempty"` - Succeeded bool `json:"succeeded,omitempty"` - Work string `json:"work,omitempty"` + Baseline *baselineRecord `json:"baseline,omitempty"` + Drain *drainObservation `json:"drain,omitempty"` + Authority *phaseAuthority `json:"authority,omitempty"` + Sequence int `json:"sequence"` + Kind string `json:"kind"` + Operation string `json:"operation,omitempty"` + ID int `json:"id,omitempty"` + SessionID string `json:"session_id,omitempty"` + RequestIDs []int64 `json:"request_ids,omitempty"` + Count int `json:"count,omitempty"` + Digest string `json:"digest,omitempty"` + Succeeded bool `json:"succeeded,omitempty"` + Work string `json:"work,omitempty"` } type Journal interface { @@ -115,6 +116,12 @@ func replay(events []Event) state { s.observedJobs[id] = true } } + if e.Operation == "drain" { + s.workObserved = true + if e.Drain == nil || e.Drain.Outcome != drainOutcomeObserved { + s.uncertain = true + } + } case "intent": if pending != "" { s.uncertain = true @@ -331,6 +338,9 @@ func (d *Driver) Run(ctx context.Context, phase string) error { return Event{}, ErrBarrier }) } + if phase == "drain" { + return d.drain(ctx, s.setID) + } return d.probe(ctx, phase, s.setID) } diff --git a/experiments/g01-scaleset/livecanary/journal.go b/experiments/g01-scaleset/livecanary/journal.go index 24b00bc..a302360 100644 --- a/experiments/g01-scaleset/livecanary/journal.go +++ b/experiments/g01-scaleset/livecanary/journal.go @@ -280,6 +280,9 @@ func validEvent(e Event) bool { case "inventory": return len(e.Digest) == 64 case "observation": + if e.Operation == "drain" { + return validDrainObservation(e.Drain) + } return e.Operation == "poll" || e.Operation == "inspect" || e.Operation == "inventory" case "response": return e.Operation == "jit" || e.Operation == "acquire" diff --git a/experiments/g01-scaleset/livecanary/response_budget.go b/experiments/g01-scaleset/livecanary/response_budget.go index 90b8677..fcadfc4 100644 --- a/experiments/g01-scaleset/livecanary/response_budget.go +++ b/experiments/g01-scaleset/livecanary/response_budget.go @@ -14,8 +14,14 @@ var errResponseBudget = errors.New("response body budget exceeded") // when it constructs session clients. Standard RegisterProtocol permits a shared // response wrapper without replacing that required type or patching the SDK. // The inner clone preserves the already configured TLS/host/proxy restrictions. -func withResponseBudget(transport *http.Transport) *http.Transport { - wrapped := responseBudgetTransport{inner: transport} +func withResponseBudget(transport *http.Transport, wrappers ...func(http.RoundTripper) http.RoundTripper) *http.Transport { + var inner http.RoundTripper = transport + for _, wrap := range wrappers { + if wrap != nil { + inner = wrap(inner) + } + } + wrapped := responseBudgetTransport{inner: inner} outer := transport.Clone() // The outer transport only dispatches to the wrapper. Disable its own HTTP/2 // setup so it cannot register a competing HTTPS handler; the inner transport diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 1ac0a65..d435dff 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -42,7 +42,7 @@ func (c Credentials) validate(a Approval, now time.Time) error { return ErrApproval } needsVerification := slices.ContainsFunc(a.Phases, func(p string) bool { - return p == "before-ack" || p == "after-ack" || p == "before-acquire" || p == "acquire-loss" + return p == "before-ack" || p == "after-ack" || p == "before-acquire" || p == "acquire-loss" || p == "drain" }) if needsVerification && (len(c.VerificationToken) < 20 || len(c.VerificationToken) > 1024 || c.VerificationToken == c.InstallationToken || strings.ContainsAny(c.VerificationToken, "\r\n\x00")) { return ErrApproval @@ -63,6 +63,10 @@ type SDKAPI struct { // Actions hosts over direct TLS. No proxy, redirect, retry logger, automatic HTTP // retry, credential environment read or worker process is configured here. func NewSDKAPI(a Approval, c Credentials) (*SDKAPI, error) { + return newSDKAPIWithPollHook(a, c, nil) +} + +func newSDKAPIWithPollHook(a Approval, c Credentials, hook *drainPollHook) (*SDKAPI, error) { if a.Validate(time.Now()) != nil || c.validate(a, time.Now()) != nil { return nil, ErrApproval } @@ -70,7 +74,14 @@ func NewSDKAPI(a Approval, c Credentials) (*SDKAPI, error) { retry := retryablehttp.NewClient() retry.RetryMax = 0 retry.Logger = nil - retry.HTTPClient = &http.Client{Transport: withResponseBudget(transport), Timeout: operationTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} + wrappers := []func(http.RoundTripper) http.RoundTripper(nil) + if hook != nil { + wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { + hook.inner = inner + return hook + }) + } + retry.HTTPClient = &http.Client{Transport: withResponseBudget(transport, wrappers...), Timeout: operationTimeout, CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} options := []scaleset.HTTPOption{scaleset.WithRetryableHTTPClint(retry), scaleset.WithLogger(slog.New(slog.DiscardHandler))} client, err := scaleset.NewClientWithPersonalAccessToken(scaleset.NewClientWithPersonalAccessTokenConfig{GitHubConfigURL: "https://github.com/" + a.Organization, PersonalAccessToken: c.InstallationToken}, options...) if err != nil { @@ -279,6 +290,24 @@ func (a *SDKAPI) DeleteScaleSet(c context.Context, id int) error { func (a *SDKAPI) OpenSession(c context.Context, id int, owner string) (Session, error) { return a.client.MessageSessionClient(c, id, owner, a.options...) } + +func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook *drainPollHook) (Session, error) { + if a == nil || hook == nil { + return nil, ErrApproval + } + configured, err := newSDKAPIWithPollHook(a.approval, a.credentials, hook) + if err != nil { + return nil, err + } + session, err := configured.client.MessageSessionClient(c, id, owner, configured.options...) + if err != nil { + return nil, ErrRemote + } + hook.mu.Lock() + hook.target = session.Session().MessageQueueURL + hook.mu.Unlock() + return session, nil +} func (a *SDKAPI) FindRunner(c context.Context, name string) (*scaleset.RunnerReference, error) { return a.client.GetRunnerByName(c, name) } diff --git a/experiments/g02-auth/broker.go b/experiments/g02-auth/broker.go index 8958714..2598ddb 100644 --- a/experiments/g02-auth/broker.go +++ b/experiments/g02-auth/broker.go @@ -49,7 +49,7 @@ type BrokerResult struct { var errBroker = errors.New("broker stopped; retain private intent and review; no automatic retry") var brokerComponent = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.-]{0,99}$`) var brokerWorkerComponent = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}$`) -var brokerPhases = map[string]bool{"create": true, "before-ack": true, "after-ack": true, "before-acquire": true, "acquire-loss": true, "jit-loss": true, "inspect": true, "cleanup": true} +var brokerPhases = map[string]bool{"create": true, "before-ack": true, "after-ack": true, "before-acquire": true, "acquire-loss": true, "jit-loss": true, "drain": true, "inspect": true, "cleanup": true} var brokerSpecialSlots = map[string]bool{"discover-actions-host": true, "paired-terminal": true} func brokerSlotAllowed(slot string) bool { return brokerPhases[slot] || brokerSpecialSlots[slot] } diff --git a/experiments/g02-auth/broker_admission_test.go b/experiments/g02-auth/broker_admission_test.go index 80ebf48..75517fe 100644 --- a/experiments/g02-auth/broker_admission_test.go +++ b/experiments/g02-auth/broker_admission_test.go @@ -108,7 +108,7 @@ func TestBrokerFinitePhasesAndUnknownRetention(t *testing.T) { func TestBrokerLedgerCapacityDerivesFromFiniteSlotSchema(t *testing.T) { want := 1 + 2*(len(brokerPhases)+len(brokerSpecialSlots)) + 1 - if got := brokerLedgerMaxLines(); got != want || got != 22 { + if got := brokerLedgerMaxLines(); got != want || got != 24 { t.Fatalf("ledger line bound=%d want schema-derived %d", got, want) } if brokerSlotAllowed("unreviewed-slot") || !brokerSlotAllowed("paired-terminal") || !brokerSlotAllowed("discover-actions-host") { diff --git a/experiments/g02-auth/broker_entry.go b/experiments/g02-auth/broker_entry.go index 7375783..dba2ef5 100644 --- a/experiments/g02-auth/broker_entry.go +++ b/experiments/g02-auth/broker_entry.go @@ -39,11 +39,11 @@ var brokerWorkflow = regexp.MustCompile(`^\.github/workflows/[a-zA-Z0-9_-]+\.ya? func (c controllerApproval) needsVerification() bool { return slices.ContainsFunc(c.Phases, func(phase string) bool { - return phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" + return phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" || phase == "drain" }) } func (c controllerApproval) validate(a BrokerApproval, now time.Time) error { - if c.OwnerNonce != a.OwnerNonce || c.AppID != a.AppID || c.InstallationID != a.InstallationID || c.Organization != a.Organization || c.Repository != a.Repository || c.RepositoryID != a.RepositoryID || c.RunnerGroupID != a.RunnerGroupID || c.HarnessSHA != a.ControllerHarnessSHA || !brokerSHA40.MatchString(c.HarnessSHA) || !brokerSHA40.MatchString(c.WorkflowSHA) || !brokerNonce.MatchString(c.OwnerNonce) || !brokerWorkflow.MatchString(c.WorkflowPath) || !brokerComponent.MatchString(c.Controller) || !c.ExpiresAt.After(now.Add(time.Minute)) || c.ExpiresAt.After(now.Add(24*time.Hour)) || c.ExpiresAt.Before(a.ExpiresAt) || len(c.ActionsHosts) < 1 || len(c.ActionsHosts) > 8 || len(c.Phases) < 1 || len(c.Phases) > 8 || c.needsVerification() != a.AllowVerificationAuthority || (c.needsVerification() && c.WorkflowRunID < 1) { + if c.OwnerNonce != a.OwnerNonce || c.AppID != a.AppID || c.InstallationID != a.InstallationID || c.Organization != a.Organization || c.Repository != a.Repository || c.RepositoryID != a.RepositoryID || c.RunnerGroupID != a.RunnerGroupID || c.HarnessSHA != a.ControllerHarnessSHA || !brokerSHA40.MatchString(c.HarnessSHA) || !brokerSHA40.MatchString(c.WorkflowSHA) || !brokerNonce.MatchString(c.OwnerNonce) || !brokerWorkflow.MatchString(c.WorkflowPath) || !brokerComponent.MatchString(c.Controller) || !c.ExpiresAt.After(now.Add(time.Minute)) || c.ExpiresAt.After(now.Add(24*time.Hour)) || c.ExpiresAt.Before(a.ExpiresAt) || len(c.ActionsHosts) < 1 || len(c.ActionsHosts) > 8 || len(c.Phases) < 1 || len(c.Phases) > 9 || c.needsVerification() != a.AllowVerificationAuthority || (c.needsVerification() && c.WorkflowRunID < 1) { return errBroker } if a.Mode == "paired-terminal" { From f44f5f9b36ede95a5244471efba6a6002164fd67 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 16:35:07 +0900 Subject: [PATCH 02/63] fix(g01): close drain lifecycle gaps --- experiments/g01-scaleset/livecanary/drain.go | 24 +++++- .../g01-scaleset/livecanary/drain_test.go | 78 +++++++++++++++++++ experiments/g01-scaleset/livecanary/driver.go | 9 ++- 3 files changed, 105 insertions(+), 6 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index a55cb28..29a25f3 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -320,7 +320,10 @@ func (b *drainHeldBody) Read(p []byte) (int, error) { <-b.release return b.source.Read(p) } -func (b *drainHeldBody) Close() error { return b.source.Close() } +func (b *drainHeldBody) Close() error { + <-b.release + return b.source.Close() +} // drainClient records only high-level listener effects and the two bounded // polls. It never changes ACK or acquisition order. @@ -488,8 +491,15 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho return obs, ErrQuarantine } hook.onRequestWritten = func() { l.SetMaxRunners(drainWithdrawnCapacity) } + runCtx, stop := context.WithCancel(ctx) + defer stop() runResult := make(chan error, 1) - go func() { runResult <- l.Run(ctx, drainScaler{client: c}) }() + go func() { runResult <- l.Run(runCtx, drainScaler{client: c}) }() + cancelAndJoin := func() { + hook.releaseResponse() + stop() + <-runResult + } release := false defer func() { @@ -521,6 +531,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho } return obs, drainBoundaryError(&obs, hook, err) case <-ctx.Done(): + cancelAndJoin() return obs, ErrQuarantine } } @@ -528,6 +539,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho case err := <-runResult: return obs, drainBoundaryError(&obs, hook, err) case <-ctx.Done(): + cancelAndJoin() return obs, ErrQuarantine } if !responseCaptured { @@ -543,10 +555,16 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho case err := <-runResult: return obs, drainBoundaryError(&obs, hook, err) case <-ctx.Done(): + cancelAndJoin() return obs, ErrQuarantine } } - err = <-runResult + select { + case err = <-runResult: + case <-ctx.Done(): + cancelAndJoin() + return obs, ErrQuarantine + } if errors.Is(err, errDrainCollected) { if obs.Boundary == "" { obs.Boundary, obs.ResponseHeld, _ = hook.boundary() diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 08e9248..fec28de 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -132,6 +132,27 @@ func TestDrainListenerMarksResponseBeforeWriteInconclusive(t *testing.T) { } } +func TestDrainHeldBodyHoldsCloseUntilRelease(t *testing.T) { + hook := newDrainPollHook("http://fixture.invalid/queue") + body := &drainHeldBody{source: http.NoBody, release: hook.release} + done := make(chan error, 1) + go func() { done <- body.Close() }() + select { + case <-done: + t.Fatal("status-only response close was not held") + case <-time.After(20 * time.Millisecond): + } + hook.releaseResponse() + select { + case err := <-done: + if err != nil { + t.Fatalf("held body close: %v", err) + } + case <-time.After(time.Second): + t.Fatal("held body close did not release") + } +} + func TestDrainObservationIsBoundedAndFailClosed(t *testing.T) { observation := validDrainTestObservation() if !validDrainObservation(&observation) { @@ -216,6 +237,34 @@ func TestDrainRequiresVerificationAuthority(t *testing.T) { } } +func TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + j := &memoryJournal{events: []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + }} + api := &drainDriverAPI{fakeAPI: &fakeAPI{}, approval: a} + d := Driver{Approval: a, Journal: j, API: api} + err := d.Run(context.Background(), "drain") + if !errors.Is(err, ErrNoMessage) { + t.Fatalf("drain result = %v, want bounded unresolved result", err) + } + if api.getScaleSetCalls != 2 { + t.Fatalf("owned set reads = %d, want before/after drain reads", api.getScaleSetCalls) + } + var observed bool + for _, event := range j.events { + if event.Kind == "observation" && event.Operation == "drain" { + observed = true + } + } + if !observed { + t.Fatal("drain did not retain its bounded observation") + } +} + func validDrainTestObservation() drainObservation { set := drainSetIdentity{ID: 7, Name: "g01-test-set", RunnerGroupID: 3, Label: "g01-test-set"} runner := &drainRunnerIdentity{ID: 19, Name: "g01-test-worker-1", ScaleSetID: 7} @@ -233,6 +282,35 @@ func validDrainTestObservation() drainObservation { } } +type drainDriverAPI struct { + *fakeAPI + approval Approval + getScaleSetCalls int +} + +func (a *drainDriverAPI) GetScaleSet(context.Context, int) (*scaleset.RunnerScaleSet, error) { + a.getScaleSetCalls++ + return &scaleset.RunnerScaleSet{ + ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, + Labels: []scaleset.Label{{Name: a.approval.setName()}}, + RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, nil +} + +func (a *drainDriverAPI) FindRunner(context.Context, string) (*scaleset.RunnerReference, error) { + return &scaleset.RunnerReference{ID: 19, Name: a.approval.workerName(), RunnerScaleSetID: 7}, nil +} + +func (*drainDriverAPI) VerifyRun(context.Context, Approval, int64) error { return nil } + +func (a *drainDriverAPI) OpenDrainSession(_ context.Context, _ int, _ string, hook *drainPollHook) (Session, error) { + hook.mu.Lock() + hook.target = "http://fixture.invalid/queue" + hook.mu.Unlock() + return &drainSyntheticSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: "http://fixture.invalid/queue", Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}}, nil +} + type drainNoTraceTransport struct { gets int } diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index e02b1d1..5441fa3 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -278,6 +278,12 @@ func (d *Driver) Run(ctx context.Context, phase string) error { if s.setID <= 0 || s.reserved { return ErrQuarantine } + if phase == "drain" { + if s.workObserved || len(s.observedJobs) != 0 { + return ErrQuarantine + } + return d.drain(ctx, s.setID) + } set, err := d.owned(ctx, s.setID) if err != nil { return err @@ -338,9 +344,6 @@ func (d *Driver) Run(ctx context.Context, phase string) error { return Event{}, ErrBarrier }) } - if phase == "drain" { - return d.drain(ctx, s.setID) - } return d.probe(ctx, phase, s.setID) } From 945371e1ec6db2a980f588d2580dfae9f5d941bd Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 16:57:34 +0900 Subject: [PATCH 03/63] fix(g01): admit drain in paired verification --- experiments/g01-scaleset/livecanary/drain_test.go | 15 +++++++++++++++ .../g01-scaleset/livecanary/paired_terminal.go | 2 +- .../g01-scaleset/livecanary/preparation.go | 2 +- .../g01-scaleset/livecanary/preparation_test.go | 5 +++++ 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index fec28de..85eb189 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -13,6 +13,7 @@ import ( "testing" "time" + "github.com/1XP-AI/gh-runnerd/experiments/g01-scaleset/liveworker" "github.com/actions/scaleset" "github.com/google/uuid" ) @@ -265,6 +266,20 @@ func TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine(t *testing.T) { } } +func TestValidatePairedApprovalsAcceptsDrainVerification(t *testing.T) { + a := approval() + a.Phases = []string{"create", "drain", "inspect", "cleanup"} + worker := liveworker.Approval{ + RunnerUpdatesDisabled: true, HarnessSHA: a.HarnessSHA, WorkflowSHA: a.WorkflowSHA, + OwnerNonce: a.OwnerNonce, Controller: a.Controller, Endpoint: "/fixture/docker.sock", + DaemonID: "fixture-daemon", ImageID: "sha256:" + strings.Repeat("a", 64), Image: liveworker.ImageReference, + ExpiresAt: a.ExpiresAt, Phases: []string{"create", "start", "inspect", "cleanup"}, + } + if err := ValidatePairedApprovals(a, worker); err != nil { + t.Fatalf("paired approval with drain verification rejected: %v", err) + } +} + func validDrainTestObservation() drainObservation { set := drainSetIdentity{ID: 7, Name: "g01-test-set", RunnerGroupID: 3, Label: "g01-test-set"} runner := &drainRunnerIdentity{ID: 19, Name: "g01-test-worker-1", ScaleSetID: 7} diff --git a/experiments/g01-scaleset/livecanary/paired_terminal.go b/experiments/g01-scaleset/livecanary/paired_terminal.go index 5aa2a16..de23eaa 100644 --- a/experiments/g01-scaleset/livecanary/paired_terminal.go +++ b/experiments/g01-scaleset/livecanary/paired_terminal.go @@ -183,7 +183,7 @@ func ValidatePairedApprovals(controller Approval, worker liveworker.Approval) er } verificationPhase := false for _, phase := range controller.Phases { - if phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" { + if phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" || phase == "drain" { verificationPhase = true break } diff --git a/experiments/g01-scaleset/livecanary/preparation.go b/experiments/g01-scaleset/livecanary/preparation.go index f20fd3e..0e34f01 100644 --- a/experiments/g01-scaleset/livecanary/preparation.go +++ b/experiments/g01-scaleset/livecanary/preparation.go @@ -101,7 +101,7 @@ func pairedPreparationReady(a Approval, now time.Time) bool { if _, ok := want[phase]; ok { want[phase] = true } - if phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" { + if phase == "before-ack" || phase == "after-ack" || phase == "before-acquire" || phase == "acquire-loss" || phase == "drain" { verification = true } } diff --git a/experiments/g01-scaleset/livecanary/preparation_test.go b/experiments/g01-scaleset/livecanary/preparation_test.go index 366fcf3..55ed7be 100644 --- a/experiments/g01-scaleset/livecanary/preparation_test.go +++ b/experiments/g01-scaleset/livecanary/preparation_test.go @@ -91,6 +91,11 @@ func TestPairedPreparationUsesDedicatedPhaseWithoutCleanupAuthority(t *testing.T if _, err := preparePairedJournal(withoutDirectory, withoutVerification, open); err == nil { t.Fatal("paired preparation accepted missing verification authority") } + withDrain := a + withDrain.Phases = []string{"create", "drain", "inspect", "cleanup"} + if !pairedPreparationReady(withDrain, time.Now()) { + t.Fatal("paired preparation rejected drain verification authority") + } } func TestPairedPreparationPreservesCanonicalControllerHistory(t *testing.T) { From 6e954cfe1aa703ec0337d318a6aa1015ed0a563e Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 17:18:23 +0900 Subject: [PATCH 04/63] fix(g01): quarantine runner identity changes --- experiments/g01-scaleset/livecanary/drain.go | 7 +++++++ experiments/g01-scaleset/livecanary/drain_driver.go | 4 ++++ experiments/g01-scaleset/livecanary/drain_test.go | 7 ++++++- 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 29a25f3..9de7f44 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -129,6 +129,10 @@ func validDrainSnapshot(s drainSnapshot, expected drainSetIdentity) bool { return true } +func sameDrainRunner(before, after *drainRunnerIdentity) bool { + return before != nil && after != nil && *before == *after +} + func validDrainObservation(o *drainObservation) bool { if o == nil || o.Version != drainObservationVersion || (o.Outcome != drainOutcomeObserved && o.Outcome != drainOutcomeInconclusive) || o.InitialCapacity != drainInitialCapacity || o.WithdrawnCapacity != drainWithdrawnCapacity || o.ServerReceipt != drainServerReceiptUnproven || o.Sequence <= 0 || o.ObservedAt.IsZero() || len(o.Ordering) > 8 { return false @@ -139,6 +143,9 @@ func validDrainObservation(o *drainObservation) bool { if o.Outcome == drainOutcomeObserved && (o.Boundary != drainBoundaryRequestWritten || !o.ResponseHeld) { return false } + if o.Outcome == drainOutcomeObserved && !sameDrainRunner(o.Before.Runner, o.After.Runner) { + return false + } seen := map[string]bool{} for _, item := range o.Ordering { if item != "poll-old" && item != "ack" && item != "acquire" && item != "poll-zero" || seen[item] { diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index dda69d4..3090b4f 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -177,6 +177,10 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return afterErr } obs.Before, obs.After = before, after + if runErr == nil && !sameDrainRunner(before.Runner, after.Runner) { + obs.Outcome = drainOutcomeInconclusive + runErr = ErrNoMessage + } if err := d.record(Event{Kind: "observation", Operation: "drain", Drain: &obs}); err != nil { return err } diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 85eb189..c714cb9 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -164,7 +164,12 @@ func TestDrainObservationIsBoundedAndFailClosed(t *testing.T) { "negative-counter": func(o *drainObservation) { o.Poll.Statistics.Available = -1 }, "negative-unknown-counter": func(o *drainObservation) { o.NextPoll.Statistics.Idle = -1; o.NextPoll.StatsKnown = false }, "identity-change": func(o *drainObservation) { o.After.Set.ID++ }, - "duplicate-order": func(o *drainObservation) { o.Ordering = append(o.Ordering, "ack") }, + "runner-change": func(o *drainObservation) { + runner := *o.After.Runner + runner.ID++ + o.After.Runner = &runner + }, + "duplicate-order": func(o *drainObservation) { o.Ordering = append(o.Ordering, "ack") }, "response-before-write-observed": func(o *drainObservation) { o.Boundary = drainBoundaryResponseBeforeWrite }, From 82d0d7d19c763fea2d0dc42b66ca239fd0b83bf4 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 17:51:34 +0900 Subject: [PATCH 05/63] fix(g01): fence drain observations and lifecycle --- .../livecanary/baseline_journal.go | 10 + experiments/g01-scaleset/livecanary/drain.go | 132 ++++++--- .../g01-scaleset/livecanary/drain_driver.go | 116 ++++++-- .../g01-scaleset/livecanary/drain_test.go | 250 +++++++++++++++++- experiments/g01-scaleset/livecanary/driver.go | 50 +++- .../g01-scaleset/livecanary/journal.go | 26 ++ experiments/g01-scaleset/livecanary/sdk.go | 24 +- .../livecanary/sdk_integration_test.go | 121 +++++++++ 8 files changed, 658 insertions(+), 71 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_journal.go b/experiments/g01-scaleset/livecanary/baseline_journal.go index ac7a956..31fadeb 100644 --- a/experiments/g01-scaleset/livecanary/baseline_journal.go +++ b/experiments/g01-scaleset/livecanary/baseline_journal.go @@ -123,6 +123,16 @@ func cloneEvent(e Event) Event { x.Ordering = slices.Clone(e.Drain.Ordering) e.Drain = &x } + if e.DrainSnapshot != nil { + data, _ := json.Marshal(e.DrainSnapshot) + var x drainSnapshot + _ = json.Unmarshal(data, &x) + if e.DrainSnapshot.Runner != nil { + runner := *e.DrainSnapshot.Runner + x.Runner = &runner + } + e.DrainSnapshot = &x + } if e.Baseline != nil { data, _ := json.Marshal(e.Baseline) var x baselineRecord diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 9de7f44..1b6b8de 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -31,6 +31,10 @@ const ( drainResponseUnknown = "unknown" drainOutcomeObserved = "observed" drainOutcomeInconclusive = "inconclusive" + drainMarkerPrerequisiteFailed = "prerequisite-failed" + drainMarkerCancelled = "cancelled" + drainMarkerDeadline = "deadline" + drainMarkerQuarantine = "quarantine" ) var errDrainCollected = errors.New("drain observation collected") @@ -52,7 +56,22 @@ func newDrainStatistics(s *scaleset.RunnerScaleSetStatistic) (drainStatistics, e if s == nil || s.TotalAvailableJobs < 0 || s.TotalAcquiredJobs < 0 || s.TotalAssignedJobs < 0 || s.TotalRunningJobs < 0 || s.TotalRegisteredRunners < 0 || s.TotalBusyRunners < 0 || s.TotalIdleRunners < 0 { return drainStatistics{}, ErrQuarantine } - return drainStatistics{Available: s.TotalAvailableJobs, Acquired: s.TotalAcquiredJobs, Assigned: s.TotalAssignedJobs, Running: s.TotalRunningJobs, Registered: s.TotalRegisteredRunners, Busy: s.TotalBusyRunners, Idle: s.TotalIdleRunners}, nil + result := drainStatistics{Available: s.TotalAvailableJobs, Acquired: s.TotalAcquiredJobs, Assigned: s.TotalAssignedJobs, Running: s.TotalRunningJobs, Registered: s.TotalRegisteredRunners, Busy: s.TotalBusyRunners, Idle: s.TotalIdleRunners} + if !validKnownDrainStatistics(result) { + return drainStatistics{}, ErrQuarantine + } + return result, nil +} + +func validKnownDrainStatistics(s drainStatistics) bool { + if s.Available < 0 || s.Acquired < 0 || s.Assigned < 0 || s.Running < 0 || s.Registered < 0 || s.Busy < 0 || s.Idle < 0 { + return false + } + // The service does not expose a separate runner state. A known snapshot is + // only useful when its bounded runner partition is internally consistent. + // Avoid Busy+Idle integer overflow by comparing one operand to the + // subtraction result. + return s.Busy <= s.Registered && s.Idle <= s.Registered-s.Busy && s.Busy+s.Idle == s.Registered } type drainPollObservation struct { @@ -80,6 +99,7 @@ type drainRunnerIdentity struct { type drainSnapshot struct { Set drainSetIdentity `json:"set"` Statistics drainStatistics `json:"statistics"` + StatsKnown bool `json:"statistics_known"` Runner *drainRunnerIdentity `json:"runner,omitempty"` } @@ -110,17 +130,17 @@ func validDrainPoll(p drainPollObservation, wantCapacity int) bool { if p.Capacity != wantCapacity || p.Message != drainMessagePresent && p.Message != drainMessageAbsent && p.Message != drainMessageUnknown || !validDrainResponse(p.ACK) || !validDrainResponse(p.Acquisition) { return false } - if p.Statistics.Available < 0 || p.Statistics.Acquired < 0 || p.Statistics.Assigned < 0 || p.Statistics.Running < 0 || p.Statistics.Registered < 0 || p.Statistics.Busy < 0 || p.Statistics.Idle < 0 { + if !p.StatsKnown && p.Statistics != (drainStatistics{}) { return false } - return true + return !p.StatsKnown || validKnownDrainStatistics(p.Statistics) } func validDrainSnapshot(s drainSnapshot, expected drainSetIdentity) bool { if s.Set != expected || s.Set.ID <= 0 || s.Set.RunnerGroupID <= 0 || s.Set.Name == "" || s.Set.Label == "" || !baselineText(s.Set.Name, 128) || !baselineText(s.Set.Label, 128) { return false } - if s.Statistics.Available < 0 || s.Statistics.Acquired < 0 || s.Statistics.Assigned < 0 || s.Statistics.Running < 0 || s.Statistics.Registered < 0 || s.Statistics.Busy < 0 || s.Statistics.Idle < 0 { + if !s.StatsKnown || !validKnownDrainStatistics(s.Statistics) { return false } if s.Runner != nil && (s.Runner.ID <= 0 || s.Runner.ScaleSetID != s.Set.ID || s.Runner.Name == "" || !baselineText(s.Runner.Name, 256)) { @@ -154,25 +174,14 @@ func validDrainObservation(o *drainObservation) bool { seen[item] = true } if o.Outcome == drainOutcomeObserved { - if o.NextPoll.Message != drainMessageAbsent || o.NextPoll.ACK != drainResponseNotAttempted || o.NextPoll.Acquisition != drainResponseNotAttempted { + if o.Poll.Message != drainMessagePresent || !o.Poll.StatsKnown || o.NextPoll.Message != drainMessageAbsent || o.NextPoll.ACK != drainResponseNotAttempted || o.NextPoll.Acquisition != drainResponseNotAttempted { return false } - if o.Poll.Message == drainMessagePresent { - if o.Poll.ACK != drainResponseSucceeded || (o.Poll.Acquisition != drainResponseNotAttempted && o.Poll.Acquisition != drainResponseSucceeded) { - return false - } - } else if o.Poll.Message == drainMessageAbsent && (o.Poll.ACK != drainResponseNotAttempted || o.Poll.Acquisition != drainResponseNotAttempted) { - return false - } else if o.Poll.Message != drainMessageAbsent { + if o.Poll.ACK != drainResponseSucceeded || o.Poll.Acquisition != drainResponseSucceeded { return false } want := []string{"poll-old"} - if o.Poll.Message == drainMessagePresent { - want = append(want, "ack") - if o.Poll.Acquisition == drainResponseSucceeded { - want = append(want, "acquire") - } - } + want = append(want, "ack", "acquire") want = append(want, "poll-zero") if !slices.Equal(o.Ordering, want) { return false @@ -243,6 +252,15 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { first := h.pollAttempts == 1 h.mu.Unlock() if !first { + h.mu.Lock() + second := h.pollAttempts == 2 + if !second { + h.invalid = true + } + h.mu.Unlock() + if !second { + return nil, ErrQuarantine + } return h.inner.RoundTrip(req) } @@ -338,15 +356,49 @@ type drainClient struct { inner listener.Client validate func(context.Context, *scaleset.RunnerScaleSetMessage) error obs *drainObservation + phaseCtx context.Context + reject func(string) error mu sync.Mutex polls int messageID int requestID int64 } +type drainContextBinder interface { + bindDrainContext(context.Context) +} + +func (c *drainClient) bindDrainContext(ctx context.Context) { + c.mu.Lock() + c.phaseCtx = ctx + c.mu.Unlock() + if binder, ok := c.inner.(drainContextBinder); ok { + binder.bindDrainContext(ctx) + } +} + +func (c *drainClient) active() bool { + c.mu.Lock() + phaseCtx := c.phaseCtx + c.mu.Unlock() + return phaseCtx == nil || phaseCtx.Err() == nil +} + +func (c *drainClient) rejectCall(operation string) error { + if c.reject != nil { + if err := c.reject(operation); err != nil { + return err + } + } + return ErrQuarantine +} + func (c *drainClient) Session() scaleset.RunnerScaleSetSession { return c.inner.Session() } func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scaleset.RunnerScaleSetMessage, error) { + if !c.active() { + return nil, c.rejectCall("observe-poll") + } c.mu.Lock() if capacity != drainInitialCapacity && capacity != drainWithdrawnCapacity || c.polls >= 2 { c.mu.Unlock() @@ -355,6 +407,8 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal c.polls++ index := c.polls c.obs.poll(index).Capacity = capacity + c.obs.poll(index).Statistics = drainStatistics{} + c.obs.poll(index).StatsKnown = false c.obs.poll(index).ACK = drainResponseNotAttempted c.obs.poll(index).Acquisition = drainResponseNotAttempted c.obs.Ordering = append(c.obs.Ordering, map[int]string{1: "poll-old", 2: "poll-zero"}[index]) @@ -374,7 +428,7 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal return nil, nil } stats, statsErr := newDrainStatistics(message.Statistics) - if statsErr != nil || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) > 1 || len(message.JobAvailableMessages) == 1 && (message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0) { + if statsErr != nil || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { c.mu.Lock() c.obs.poll(index).Message = drainMessageUnknown c.mu.Unlock() @@ -409,13 +463,19 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal } func (c *drainClient) DeleteMessage(ctx context.Context, id int) error { - err := c.inner.DeleteMessage(ctx, id) + if !c.active() { + return c.rejectCall("ack") + } c.mu.Lock() - defer c.mu.Unlock() index := c.polls - if index < 1 || index > 2 || c.messageID <= 0 || id != c.messageID { - return ErrQuarantine + valid := index == 1 && c.messageID > 0 && id == c.messageID && c.obs.poll(index).ACK == drainResponseNotAttempted + c.mu.Unlock() + if !valid { + return c.rejectCall("ack") } + err := c.inner.DeleteMessage(ctx, id) + c.mu.Lock() + defer c.mu.Unlock() c.obs.poll(index).ACK = drainResponseSucceeded c.obs.Ordering = append(c.obs.Ordering, "ack") if err != nil { @@ -425,16 +485,19 @@ func (c *drainClient) DeleteMessage(ctx context.Context, id int) error { } func (c *drainClient) AcquireJobs(ctx context.Context, ids []int64) ([]int64, error) { - if len(ids) != 1 { - return nil, ErrQuarantine + if !c.active() { + return nil, c.rejectCall("acquire") } - got, err := c.inner.AcquireJobs(ctx, slices.Clone(ids)) c.mu.Lock() - defer c.mu.Unlock() index := c.polls - if index < 1 || index > 2 || c.requestID <= 0 || ids[0] != c.requestID { - return nil, ErrQuarantine + valid := index == 1 && len(ids) == 1 && c.requestID > 0 && ids[0] == c.requestID && c.obs.poll(index).ACK == drainResponseSucceeded && c.obs.poll(index).Acquisition == drainResponseNotAttempted + c.mu.Unlock() + if !valid { + return nil, c.rejectCall("acquire") } + got, err := c.inner.AcquireJobs(ctx, slices.Clone(ids)) + c.mu.Lock() + defer c.mu.Unlock() c.obs.poll(index).Acquisition = drainResponseSucceeded c.obs.Ordering = append(c.obs.Ordering, "acquire") if err != nil { @@ -457,6 +520,9 @@ func (s drainScaler) HandleDesiredRunnerCount(_ context.Context, count int) (int s.client.mu.Lock() polls := s.client.polls s.client.mu.Unlock() + if !s.client.active() { + return 0, s.client.rejectCall("observe-poll") + } if polls >= 2 { return 0, errDrainCollected } @@ -493,6 +559,9 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho obs.Poll.StatsKnown = true } c := &drainClient{inner: client, obs: &obs} + if rejecter, ok := client.(interface{ reject(string) error }); ok { + c.reject = rejecter.reject + } l, err := listener.New(c, listener.Config{ScaleSetID: setID, MaxRunners: drainInitialCapacity}) if err != nil { return obs, ErrQuarantine @@ -500,6 +569,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho hook.onRequestWritten = func() { l.SetMaxRunners(drainWithdrawnCapacity) } runCtx, stop := context.WithCancel(ctx) defer stop() + c.bindDrainContext(runCtx) runResult := make(chan error, 1) go func() { runResult <- l.Run(runCtx, drainScaler{client: c}) }() cancelAndJoin := func() { @@ -576,7 +646,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho if obs.Boundary == "" { obs.Boundary, obs.ResponseHeld, _ = hook.boundary() } - if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld { + if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld && obs.Poll.Message == drainMessagePresent && obs.Poll.StatsKnown && obs.Poll.ACK == drainResponseSucceeded && obs.Poll.Acquisition == drainResponseSucceeded && obs.NextPoll.Message == drainMessageAbsent { obs.Outcome = drainOutcomeObserved return obs, nil } @@ -596,7 +666,7 @@ func drainBoundaryError(obs *drainObservation, hook *drainPollHook, err error) e if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return ErrQuarantine } - if obs != nil && obs.Boundary == drainBoundaryUnresolved { + if obs != nil && (obs.Boundary == drainBoundaryUnresolved || obs.Boundary == drainBoundaryResponseBeforeWrite) { return ErrNoMessage } return ErrQuarantine diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 3090b4f..be9ccbf 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -16,22 +16,52 @@ type journaledDrainClient struct { d *Driver inner Session sessionID string + phaseCtx context.Context +} + +func (c *journaledDrainClient) bindDrainContext(ctx context.Context) { c.phaseCtx = ctx } + +func (c *journaledDrainClient) operationContext(fallback context.Context) (context.Context, error) { + if c.phaseCtx != nil { + if c.phaseCtx.Err() != nil { + return nil, ErrQuarantine + } + return c.phaseCtx, nil + } + if fallback == nil || fallback.Err() != nil { + return nil, ErrQuarantine + } + return fallback, nil +} + +func (c *journaledDrainClient) reject(operation string) error { + if c.d.record(Event{Kind: "unknown", Operation: operation}) != nil { + return ErrJournal + } + return ErrQuarantine } func (c *journaledDrainClient) Session() scaleset.RunnerScaleSetSession { return c.inner.Session() } func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity int) (*scaleset.RunnerScaleSetMessage, error) { + callCtx, err := c.operationContext(ctx) + if err != nil { + return nil, err + } var message *scaleset.RunnerScaleSetMessage - err := c.d.effect(ctx, "observe-poll", nil, func(call context.Context) (Event, error) { - var err error - message, err = c.inner.GetMessage(call, last, capacity) - if err != nil { - return Event{}, err + err = c.d.effect(callCtx, "observe-poll", nil, func(call context.Context) (Event, error) { + var callErr error + message, callErr = c.inner.GetMessage(call, last, capacity) + if callErr != nil { + return Event{}, callErr } if message == nil { return Event{SessionID: c.sessionID}, nil } - if message.MessageID <= 0 || len(message.JobAvailableMessages) > 1 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || message.Statistics == nil { + if message.MessageID <= 0 || len(message.JobAvailableMessages) != 1 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || message.Statistics == nil { + return Event{}, ErrRemote + } + if _, callErr = newDrainStatistics(message.Statistics); callErr != nil { return Event{}, ErrRemote } for _, job := range message.JobAvailableMessages { @@ -44,7 +74,8 @@ func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity in return Event{}, ErrApproval } } - return Event{ID: message.MessageID, SessionID: c.sessionID}, nil + requestID := message.JobAvailableMessages[0].RunnerRequestID + return Event{ID: message.MessageID, SessionID: c.sessionID, RequestIDs: []int64{requestID}, Work: workDemand}, nil }) if err != nil { return nil, err @@ -53,7 +84,11 @@ func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity in } func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error { - return c.d.effect(ctx, "ack", nil, func(call context.Context) (Event, error) { + callCtx, err := c.operationContext(ctx) + if err != nil { + return err + } + return c.d.effect(callCtx, "ack", nil, func(call context.Context) (Event, error) { if err := c.inner.DeleteMessage(call, id); err != nil { return Event{}, err } @@ -62,9 +97,13 @@ func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error } func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([]int64, error) { + callCtx, err := c.operationContext(ctx) + if err != nil { + return nil, err + } ids = slices.Clone(ids) var got []int64 - err := c.d.effect(ctx, "acquire", ids, func(call context.Context) (Event, error) { + err = c.d.effect(callCtx, "acquire", ids, func(call context.Context) (Event, error) { var err error got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) if err != nil || !slices.Equal(got, ids) { @@ -99,6 +138,7 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr } snapshot.Set = drainSetIdentity{ID: set.ID, Name: set.Name, RunnerGroupID: set.RunnerGroupID, Label: label} snapshot.Statistics = stats + snapshot.StatsKnown = true return Event{ID: set.ID}, nil }); err != nil { return drainSnapshot{}, err @@ -111,13 +151,13 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr return Event{}, err } if runner == nil { - return Event{}, nil + return Event{DrainSnapshot: &snapshot}, nil } if runner.ID <= 0 || runner.Name != d.Approval.workerName() || runner.RunnerScaleSetID != setID { - return Event{Work: workUnresolved}, nil + return Event{DrainSnapshot: &snapshot, Work: workUnresolved}, nil } snapshot.Runner = &drainRunnerIdentity{ID: runner.ID, Name: runner.Name, ScaleSetID: runner.RunnerScaleSetID} - return Event{ID: runner.ID}, nil + return Event{ID: runner.ID, DrainSnapshot: &snapshot}, nil }); err != nil { return drainSnapshot{}, err } @@ -126,7 +166,29 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr } func validDrainIdlePrerequisite(s drainSnapshot) bool { - return s.Runner != nil && s.Statistics.Available == 0 && s.Statistics.Acquired == 0 && s.Statistics.Assigned == 0 && s.Statistics.Running == 0 && s.Statistics.Registered == 1 && s.Statistics.Busy == 0 && s.Statistics.Idle == 1 + return s.StatsKnown && s.Runner != nil && s.Statistics.Available == 0 && s.Statistics.Acquired == 0 && s.Statistics.Assigned == 0 && s.Statistics.Running == 0 && s.Statistics.Registered == 1 && s.Statistics.Busy == 0 && s.Statistics.Idle == 1 +} + +func drainMarkerFor(ctx context.Context, err error) string { + if ctx != nil { + switch ctx.Err() { + case context.DeadlineExceeded: + return drainMarkerDeadline + case context.Canceled: + return drainMarkerCancelled + } + } + if errors.Is(err, ErrQuarantine) || errors.Is(err, ErrNoMessage) { + return drainMarkerQuarantine + } + return drainMarkerQuarantine +} + +func (d *Driver) recordDrainMarker(marker string) error { + if marker == "" { + marker = drainMarkerQuarantine + } + return d.record(Event{Kind: "observation", Operation: "drain-marker", DrainMarker: marker}) } func (d *Driver) drain(ctx context.Context, setID int) error { @@ -135,7 +197,13 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return ErrApproval } before, err := d.drainSnapshot(ctx, setID, "before") - if err != nil || !validDrainIdlePrerequisite(before) { + if err != nil { + return err + } + if !validDrainIdlePrerequisite(before) { + if markerErr := d.recordDrainMarker(drainMarkerPrerequisiteFailed); markerErr != nil { + return markerErr + } return ErrQuarantine } hook := newDrainPollHook("") @@ -148,7 +216,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return Event{}, ErrRemote } current := session.Session() - if current.SessionID == [16]byte{} || current.OwnerName != d.Approval.setName() || current.Statistics == nil { + if current.SessionID == [16]byte{} || current.OwnerName != d.Approval.setName() || current.RunnerScaleSet == nil || current.RunnerScaleSet.ID != setID || current.RunnerScaleSet.Name != d.Approval.setName() || current.RunnerScaleSet.RunnerGroupID != d.Approval.RunnerGroupID || !current.RunnerScaleSet.RunnerSetting.DisableUpdate || !slices.ContainsFunc(current.RunnerScaleSet.Labels, func(label scaleset.Label) bool { return label.Name == d.Approval.setName() }) || current.Statistics == nil { return Event{}, ErrQuarantine } stats, err := newDrainStatistics(current.Statistics) @@ -159,6 +227,9 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return Event{SessionID: sessionID}, nil }) if err != nil { + if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, err)); markerErr != nil { + return markerErr + } return err } journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID} @@ -169,11 +240,26 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return Event{SessionID: sessionID}, nil }) + if ctx.Err() != nil { + if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, runErr)); markerErr != nil { + return markerErr + } + if runErr != nil { + return runErr + } + return ErrQuarantine + } if closeErr != nil { + if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, closeErr)); markerErr != nil { + return markerErr + } return closeErr } after, afterErr := d.drainSnapshot(ctx, setID, "after") if afterErr != nil { + if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, afterErr)); markerErr != nil { + return markerErr + } return afterErr } obs.Before, obs.After = before, after diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index c714cb9..564cca7 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -133,6 +133,32 @@ func TestDrainListenerMarksResponseBeforeWriteInconclusive(t *testing.T) { } } +func TestDrainListenerNoMessageIsInconclusive(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.WriteHeader(http.StatusAccepted) + return + } + http.NotFound(w, r) + })) + defer server.Close() + hook := newDrainPollHook(server.URL) + session := &drainSyntheticSession{ + client: http.DefaultClient, + initial: scaleset.RunnerScaleSetSession{ + SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: server.URL, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, + } + client := *http.DefaultClient + client.Transport = hook + session.client = &client + observation, err := runDrainListener(context.Background(), session, 7, hook) + if !errors.Is(err, ErrNoMessage) || observation.Outcome != drainOutcomeInconclusive || observation.Poll.Message != drainMessageAbsent { + t.Fatalf("no-message drain = %+v err=%v, want inconclusive", observation, err) + } +} + func TestDrainHeldBodyHoldsCloseUntilRelease(t *testing.T) { hook := newDrainPollHook("http://fixture.invalid/queue") body := &drainHeldBody{source: http.NoBody, release: hook.release} @@ -243,6 +269,170 @@ func TestDrainRequiresVerificationAuthority(t *testing.T) { } } +func TestDrainPhaseAuthorityIsAccepted(t *testing.T) { + a := approval() + a.Phases = []string{"create", "drain", "inspect", "cleanup"} + if err := a.Validate(time.Now()); err != nil { + t.Fatalf("issue-71 drain authority rejected: %v", err) + } +} + +func TestDrainPollJournalsReservationBeforeACK(t *testing.T) { + a := approval() + j := &memoryJournal{} + d := Driver{Approval: a, Journal: j, API: &fakeAPI{}} + inner := &drainGuardSession{ + initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.setName(), Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}, + message: &scaleset.RunnerScaleSetMessage{ + MessageID: 17, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalAvailableJobs: 1, TotalAssignedJobs: 1, TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + JobAvailableMessages: []*scaleset.JobAvailable{{JobMessageBase: scaleset.JobMessageBase{RunnerRequestID: 41, WorkflowRunID: a.WorkflowRunID, OwnerName: a.Organization, RepositoryName: a.Repository}}}, + }, + } + client := &journaledDrainClient{d: &d, inner: inner, sessionID: inner.initial.SessionID.String()} + if _, err := client.GetMessage(context.Background(), 0, drainInitialCapacity); err != nil { + t.Fatalf("journaled poll: %v", err) + } + var poll Event + for _, event := range j.Events() { + if event.Kind == "result" && event.Operation == "observe-poll" { + poll = event + } + } + if len(poll.RequestIDs) != 1 || poll.RequestIDs[0] != 41 || poll.Work != workDemand { + t.Fatalf("poll reservation = %+v, want request 41/demand", poll) + } + state := replay(j.Events()) + if !state.reserved || !state.workObserved || !state.observedJobs[41] { + t.Fatalf("poll reservation replay = %+v, want durable quarantine fence", state) + } +} + +func TestDrainRejectedIdlePrerequisiteRetainsFence(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + j := &memoryJournal{events: []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + }} + api := &drainDriverAPI{fakeAPI: &fakeAPI{}, approval: a, snapshotStats: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalBusyRunners: 1}} + d := Driver{Approval: a, Journal: j, API: api} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("busy idle prerequisite = %v, want quarantine", err) + } + state := replay(j.Events()) + if !state.uncertain { + t.Fatalf("rejected prerequisite replay = %+v, want durable uncertainty", state) + } + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain-marker" && event.DrainMarker == drainMarkerPrerequisiteFailed { + return + } + } + t.Fatal("rejected prerequisite did not persist a bounded marker") +} + +func TestDrainRejectsEffectsAfterCancellationAndRecordsMarker(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + j := &memoryJournal{events: []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + }} + api := &drainDriverAPI{fakeAPI: &fakeAPI{}, approval: a, blocking: true, opened: make(chan struct{})} + d := Driver{Approval: a, Journal: j, API: api} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan error, 1) + go func() { done <- d.Run(ctx, "drain") }() + select { + case <-api.opened: + cancel() + case <-time.After(time.Second): + t.Fatal("drain session did not open") + } + select { + case err := <-done: + if !errors.Is(err, ErrQuarantine) { + t.Fatalf("canceled drain = %v, want quarantine", err) + } + case <-time.After(2 * time.Second): + t.Fatal("canceled drain did not join listener") + } + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain-marker" && event.DrainMarker == drainMarkerCancelled { + return + } + } + t.Fatal("canceled drain did not persist cancellation marker") +} + +func TestDrainRejectsDuplicateOrWrongEffectsBeforeInnerCall(t *testing.T) { + inner := &drainGuardSession{ + initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: "fixture", Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}, + message: &scaleset.RunnerScaleSetMessage{ + MessageID: 7, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalAvailableJobs: 1, TotalAssignedJobs: 1, TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + JobAvailableMessages: []*scaleset.JobAvailable{{JobMessageBase: scaleset.JobMessageBase{RunnerRequestID: 11}}}, + }, + } + obs := drainObservation{Poll: drainPollObservation{Message: drainMessageUnknown}, NextPoll: drainPollObservation{Message: drainMessageUnknown}} + client := &drainClient{inner: inner, obs: &obs, phaseCtx: context.Background()} + if _, err := client.GetMessage(context.Background(), 0, drainInitialCapacity); err != nil { + t.Fatalf("guard poll: %v", err) + } + if err := client.DeleteMessage(context.Background(), 8); !errors.Is(err, ErrQuarantine) || inner.ackCalls != 0 { + t.Fatalf("wrong ACK = %v calls=%d, want pre-effect quarantine", err, inner.ackCalls) + } + if err := client.DeleteMessage(context.Background(), 7); err != nil || inner.ackCalls != 1 { + t.Fatalf("valid ACK = %v calls=%d", err, inner.ackCalls) + } + if err := client.DeleteMessage(context.Background(), 7); !errors.Is(err, ErrQuarantine) || inner.ackCalls != 1 { + t.Fatalf("duplicate ACK = %v calls=%d, want no second call", err, inner.ackCalls) + } + if _, err := client.AcquireJobs(context.Background(), []int64{12}); !errors.Is(err, ErrQuarantine) || inner.acquireCalls != 0 { + t.Fatalf("wrong acquire = %v calls=%d, want pre-effect quarantine", err, inner.acquireCalls) + } + if _, err := client.AcquireJobs(context.Background(), []int64{11}); err != nil || inner.acquireCalls != 1 { + t.Fatalf("valid acquire = %v calls=%d", err, inner.acquireCalls) + } + if _, err := client.AcquireJobs(context.Background(), []int64{11}); !errors.Is(err, ErrQuarantine) || inner.acquireCalls != 1 { + t.Fatalf("duplicate acquire = %v calls=%d, want no second call", err, inner.acquireCalls) + } +} + +func TestDrainRequiresKnownConsistentStatisticsAndControlledMessage(t *testing.T) { + observation := validDrainTestObservation() + unknown := observation + unknown.Poll.StatsKnown = false + unknown.Poll.Statistics = drainStatistics{} + if validDrainObservation(&unknown) { + t.Fatal("observed drain with unknown poll statistics accepted") + } + contradictory := observation + contradictory.Poll.Statistics.Registered = 1 + contradictory.Poll.Statistics.Busy = 1 + contradictory.Poll.Statistics.Idle = 1 + if validDrainObservation(&contradictory) { + t.Fatal("contradictory runner partition accepted") + } + if _, err := newDrainStatistics(&scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalBusyRunners: 1, TotalIdleRunners: 1}); !errors.Is(err, ErrQuarantine) { + t.Fatalf("contradictory source statistics = %v, want quarantine", err) + } + noMessage := observation + noMessage.Poll.Message = drainMessageAbsent + noMessage.Poll.Statistics = drainStatistics{} + noMessage.Poll.StatsKnown = false + noMessage.Poll.ACK = drainResponseNotAttempted + noMessage.Poll.Acquisition = drainResponseNotAttempted + noMessage.Ordering = []string{"poll-old", "poll-zero"} + if validDrainObservation(&noMessage) { + t.Fatal("observed drain with no controlled old message accepted") + } +} + func TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine(t *testing.T) { a := approval() a.Phases = append(a.Phases, "drain") @@ -296,8 +486,8 @@ func validDrainTestObservation() drainObservation { ResponseHeld: true, Poll: drainPollObservation{Capacity: drainInitialCapacity, Message: drainMessagePresent, Statistics: drainStatistics{Available: 1, Assigned: 1, Registered: 1, Idle: 1}, StatsKnown: true, ACK: drainResponseSucceeded, Acquisition: drainResponseSucceeded}, NextPoll: drainPollObservation{Capacity: drainWithdrawnCapacity, Message: drainMessageAbsent, Statistics: beforeStats, StatsKnown: true, ACK: drainResponseNotAttempted, Acquisition: drainResponseNotAttempted}, - Before: drainSnapshot{Set: set, Statistics: beforeStats, Runner: runner}, - After: drainSnapshot{Set: set, Statistics: beforeStats, Runner: runner}, + Before: drainSnapshot{Set: set, Statistics: beforeStats, StatsKnown: true, Runner: runner}, + After: drainSnapshot{Set: set, Statistics: beforeStats, StatsKnown: true, Runner: runner}, Ordering: []string{"poll-old", "ack", "acquire", "poll-zero"}, Sequence: 1, ObservedAt: time.Unix(1, 0).UTC(), } } @@ -306,15 +496,22 @@ type drainDriverAPI struct { *fakeAPI approval Approval getScaleSetCalls int + snapshotStats *scaleset.RunnerScaleSetStatistic + blocking bool + opened chan struct{} } func (a *drainDriverAPI) GetScaleSet(context.Context, int) (*scaleset.RunnerScaleSet, error) { a.getScaleSetCalls++ + stats := a.snapshotStats + if stats == nil { + stats = &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1} + } return &scaleset.RunnerScaleSet{ ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, Labels: []scaleset.Label{{Name: a.approval.setName()}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, - Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + Statistics: stats, }, nil } @@ -325,10 +522,55 @@ func (a *drainDriverAPI) FindRunner(context.Context, string) (*scaleset.RunnerRe func (*drainDriverAPI) VerifyRun(context.Context, Approval, int64) error { return nil } func (a *drainDriverAPI) OpenDrainSession(_ context.Context, _ int, _ string, hook *drainPollHook) (Session, error) { + if a.blocking { + close(a.opened) + hook.mu.Lock() + hook.target = "http://fixture.invalid/queue" + hook.mu.Unlock() + return &drainBlockingSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: hook.target, RunnerScaleSet: &scaleset.RunnerScaleSet{ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, Labels: []scaleset.Label{{Name: a.approval.setName()}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}}, nil + } hook.mu.Lock() hook.target = "http://fixture.invalid/queue" hook.mu.Unlock() - return &drainSyntheticSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: "http://fixture.invalid/queue", Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}}, nil + return &drainSyntheticSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: "http://fixture.invalid/queue", RunnerScaleSet: &scaleset.RunnerScaleSet{ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, Labels: []scaleset.Label{{Name: a.approval.setName()}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}}, nil +} + +type drainGuardSession struct { + initial scaleset.RunnerScaleSetSession + message *scaleset.RunnerScaleSetMessage + ackCalls int + acquireCalls int +} + +func (s *drainGuardSession) Session() scaleset.RunnerScaleSetSession { return s.initial } +func (s *drainGuardSession) Close(context.Context) error { return nil } +func (s *drainGuardSession) GetMessage(context.Context, int, int) (*scaleset.RunnerScaleSetMessage, error) { + return s.message, nil +} +func (s *drainGuardSession) DeleteMessage(context.Context, int) error { + s.ackCalls++ + return nil +} +func (s *drainGuardSession) AcquireJobs(_ context.Context, ids []int64) ([]int64, error) { + s.acquireCalls++ + return append([]int64(nil), ids...), nil +} + +type drainBlockingSession struct { + initial scaleset.RunnerScaleSetSession +} + +func (s *drainBlockingSession) Session() scaleset.RunnerScaleSetSession { return s.initial } +func (s *drainBlockingSession) Close(ctx context.Context) error { return ctx.Err() } +func (s *drainBlockingSession) GetMessage(ctx context.Context, _, _ int) (*scaleset.RunnerScaleSetMessage, error) { + <-ctx.Done() + return nil, ctx.Err() +} +func (s *drainBlockingSession) DeleteMessage(context.Context, int) error { + return errors.New("unexpected ACK") +} +func (s *drainBlockingSession) AcquireJobs(context.Context, []int64) ([]int64, error) { + return nil, errors.New("unexpected acquisition") } type drainNoTraceTransport struct { diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index 5441fa3..940043d 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -41,19 +41,21 @@ type Approval struct { } type Event struct { - Baseline *baselineRecord `json:"baseline,omitempty"` - Drain *drainObservation `json:"drain,omitempty"` - Authority *phaseAuthority `json:"authority,omitempty"` - Sequence int `json:"sequence"` - Kind string `json:"kind"` - Operation string `json:"operation,omitempty"` - ID int `json:"id,omitempty"` - SessionID string `json:"session_id,omitempty"` - RequestIDs []int64 `json:"request_ids,omitempty"` - Count int `json:"count,omitempty"` - Digest string `json:"digest,omitempty"` - Succeeded bool `json:"succeeded,omitempty"` - Work string `json:"work,omitempty"` + Baseline *baselineRecord `json:"baseline,omitempty"` + Drain *drainObservation `json:"drain,omitempty"` + DrainSnapshot *drainSnapshot `json:"drain_snapshot,omitempty"` + DrainMarker string `json:"drain_marker,omitempty"` + Authority *phaseAuthority `json:"authority,omitempty"` + Sequence int `json:"sequence"` + Kind string `json:"kind"` + Operation string `json:"operation,omitempty"` + ID int `json:"id,omitempty"` + SessionID string `json:"session_id,omitempty"` + RequestIDs []int64 `json:"request_ids,omitempty"` + Count int `json:"count,omitempty"` + Digest string `json:"digest,omitempty"` + Succeeded bool `json:"succeeded,omitempty"` + Work string `json:"work,omitempty"` } type Journal interface { @@ -111,17 +113,27 @@ func replay(events []Event) state { case "inventory": s.inventory = e.Digest case "observation": - if e.Operation == "poll" { + if e.Operation == "poll" || e.Operation == "drain" { for _, id := range e.RequestIDs { s.observedJobs[id] = true } } + if len(e.RequestIDs) > 0 { + s.reserved = true + s.workObserved = true + } if e.Operation == "drain" { s.workObserved = true if e.Drain == nil || e.Drain.Outcome != drainOutcomeObserved { s.uncertain = true } } + if e.Operation == "drain-marker" { + s.uncertain = true + } + if e.DrainSnapshot != nil && !validDrainIdlePrerequisite(*e.DrainSnapshot) { + s.uncertain = true + } case "intent": if pending != "" { s.uncertain = true @@ -139,6 +151,16 @@ func replay(events []Event) state { s.workObserved = true s.uncertain = s.uncertain || e.Work == workUnresolved } + if len(e.RequestIDs) > 0 { + s.reserved = true + s.workObserved = true + for _, id := range e.RequestIDs { + s.observedJobs[id] = true + } + } + if e.DrainSnapshot != nil && !validDrainIdlePrerequisite(*e.DrainSnapshot) { + s.uncertain = true + } switch e.Operation { case "create": s.setID = e.ID diff --git a/experiments/g01-scaleset/livecanary/journal.go b/experiments/g01-scaleset/livecanary/journal.go index a302360..19d9b1f 100644 --- a/experiments/g01-scaleset/livecanary/journal.go +++ b/experiments/g01-scaleset/livecanary/journal.go @@ -259,6 +259,14 @@ func openJournalAtAdmission(directory string, a Approval, admissionDirectory str } func validEvent(e Event) bool { + if e.Operation == "observe-poll" && len(e.RequestIDs) > 1 { + return false + } + for _, id := range e.RequestIDs { + if id <= 0 { + return false + } + } if e.Baseline != nil || e.Kind == "baseline" { return validBaselineShape(e) } @@ -272,6 +280,21 @@ func validEvent(e Event) bool { return false } } + if e.DrainSnapshot != nil { + if e.Kind != "result" || e.Operation != "observe-runner" || !validDrainSnapshot(*e.DrainSnapshot, e.DrainSnapshot.Set) { + return false + } + } + if e.DrainMarker != "" { + if e.Kind != "observation" || e.Operation != "drain-marker" { + return false + } + switch e.DrainMarker { + case drainMarkerPrerequisiteFailed, drainMarkerCancelled, drainMarkerDeadline, drainMarkerQuarantine: + default: + return false + } + } switch e.Kind { case "authority": return e.Authority != nil @@ -283,6 +306,9 @@ func validEvent(e Event) bool { if e.Operation == "drain" { return validDrainObservation(e.Drain) } + if e.Operation == "drain-marker" { + return e.DrainMarker != "" + } return e.Operation == "poll" || e.Operation == "inspect" || e.Operation == "inventory" case "response": return e.Operation == "jit" || e.Operation == "acquire" diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index d435dff..10632b5 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -51,12 +51,16 @@ func (c Credentials) validate(a Approval, now time.Time) error { } type SDKAPI struct { - client *scaleset.Client - rest *http.Client - baseURL string - approval Approval - credentials Credentials - options []scaleset.HTTPOption + client *scaleset.Client + rest *http.Client + baseURL string + approval Approval + credentials Credentials + options []scaleset.HTTPOption + // drainClientFactory is nil in production. Tests use it only to bind the + // pinned SDK client to an offline loopback transport while still exercising + // OpenDrainSession and MessageSessionClient together. + drainClientFactory func(*drainPollHook) (*SDKAPI, error) } // NewSDKAPI is network-lazy. It permits only api.github.com and exact approved @@ -295,7 +299,13 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook if a == nil || hook == nil { return nil, ErrApproval } - configured, err := newSDKAPIWithPollHook(a.approval, a.credentials, hook) + var configured *SDKAPI + var err error + if a.drainClientFactory != nil { + configured, err = a.drainClientFactory(hook) + } else { + configured, err = newSDKAPIWithPollHook(a.approval, a.credentials, hook) + } if err != nil { return nil, err } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 17bf4c4..6f19691 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "log/slog" "net" "net/http" "net/http/httptest" @@ -175,3 +176,123 @@ func TestDriverBarriersThroughPinnedSDK(t *testing.T) { }) } } + +func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + set := &scaleset.RunnerScaleSet{ + ID: 7, Name: a.setName(), RunnerGroupID: a.RunnerGroupID, + Labels: []scaleset.Label{{Name: a.setName()}}, + RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + } + var polls, acks, acquires, closes int + var server *httptest.Server + server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/actions/runners/registration-token") && r.Method == http.MethodPost: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{"token": "fixture-admin"}) + case strings.HasSuffix(r.URL.Path, "/actions/runner-registration") && r.Method == http.MethodPost: + claims, _ := json.Marshal(map[string]int64{"exp": time.Now().Add(time.Hour).Unix()}) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{"url": server.URL, "token": "eyJhbGciOiJub25lIn0." + base64.RawURLEncoding.EncodeToString(claims) + "."}) + case strings.HasSuffix(r.URL.Path, "/sessions") && r.Method == http.MethodPost: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(scaleset.RunnerScaleSetSession{ + SessionID: uuid.MustParse("00000000-0000-4000-8000-000000000011"), OwnerName: a.setName(), + MessageQueueURL: server.URL + "/queue", MessageQueueAccessToken: "fixture-queue", + RunnerScaleSet: set, Statistics: set.Statistics, + }) + case r.URL.Path == "/queue" && r.Method == http.MethodGet: + if polls == 0 { + if r.Header.Get(scaleset.HeaderScaleSetMaxCapacity) != "1" { + t.Errorf("first poll capacity = %q, want 1", r.Header.Get(scaleset.HeaderScaleSetMaxCapacity)) + } + polls++ + jobs, _ := json.Marshal([]any{map[string]any{"messageType": "JobAvailable", "runnerRequestId": 41, "workflowRunId": a.WorkflowRunID, "ownerName": a.Organization, "repositoryName": a.Repository}}) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "messageId": 17, "messageType": "RunnerScaleSetJobMessages", "body": string(jobs), + "statistics": map[string]int{"totalAvailableJobs": 1, "totalAssignedJobs": 1, "totalRegisteredRunners": 1, "totalIdleRunners": 1}, + }) + return + } + if r.Header.Get(scaleset.HeaderScaleSetMaxCapacity) != "0" { + t.Errorf("next poll capacity = %q, want 0", r.Header.Get(scaleset.HeaderScaleSetMaxCapacity)) + } + polls++ + w.WriteHeader(http.StatusAccepted) + case r.URL.Path == "/queue/17" && r.Method == http.MethodDelete: + acks++ + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/acquirejobs") && r.Method == http.MethodPost: + acquires++ + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []int64{41}}) + case strings.Contains(r.URL.Path, "/sessions/") && r.Method == http.MethodDelete: + closes++ + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/runnerscalesets/7") && r.Method == http.MethodGet: + _ = json.NewEncoder(w).Encode(set) + case strings.HasSuffix(r.URL.Path, "/agents") && r.Method == http.MethodGet: + _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []any{map[string]any{"id": 19, "name": a.workerName(), "runnerScaleSetId": 7}}}) + default: + http.NotFound(w, r) + } + })) + defer server.Close() + + buildAPI := func(hook *drainPollHook) (*SDKAPI, error) { + retry := retryablehttp.NewClient() + retry.RetryMax = 0 + retry.Logger = nil + retry.HTTPClient.Timeout = time.Second + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address != server.Listener.Addr().String() { + return nil, errors.New("non-fixture address denied") + } + return (&net.Dialer{}).DialContext(ctx, network, address) + } + wrappers := []func(http.RoundTripper) http.RoundTripper(nil) + if hook != nil { + wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { + hook.inner = inner + return hook + }) + } + retry.HTTPClient.Transport = withResponseBudget(transport, wrappers...) + options := []scaleset.HTTPOption{scaleset.WithRetryableHTTPClint(retry), scaleset.WithLogger(slog.New(slog.DiscardHandler))} + client, err := scaleset.NewClientWithPersonalAccessToken(scaleset.NewClientWithPersonalAccessTokenConfig{GitHubConfigURL: server.URL + "/fixture-org", PersonalAccessToken: "synthetic-installation"}, options...) + if err != nil { + return nil, err + } + return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: server.URL + "/api/v3", approval: a, options: options}, nil + } + base, err := buildAPI(nil) + if err != nil { + t.Fatal(err) + } + base.drainClientFactory = buildAPI + api := fixtureSDK{base} + j := &memoryJournal{events: []Event{{Kind: "phase", Operation: "create"}, {Kind: "intent", Operation: "create"}, {Kind: "result", Operation: "create", ID: 7}}} + d := Driver{Approval: a, Journal: j, API: api} + if err := d.Run(context.Background(), "drain"); err != nil { + t.Fatalf("pinned SDK drain: %v", err) + } + if polls != 2 || acks != 1 || acquires != 1 || closes != 1 { + t.Fatalf("pinned SDK effects polls=%d ack=%d acquire=%d close=%d", polls, acks, acquires, closes) + } + var observed bool + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { + observed = true + } + } + if !observed { + t.Fatal("pinned SDK drain did not retain observed outcome") + } +} From c6775988869e136045e2932a99ca0180f10ce93a Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 17:55:06 +0900 Subject: [PATCH 06/63] docs(g01): record independent drain corrections --- docs/evidence/g01-idle-drain.md | 72 +++++++++++++++++++++++++-------- 1 file changed, 55 insertions(+), 17 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 673ce7a..c2b4f06 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -7,9 +7,11 @@ parent G01 goal, add a production provider, or authorize live execution. The invariant is: withdrawing listener capacity to zero may affect the next poll, while the poll already admitted by the listener must retain the pinned -SDK's ACK-before-acquisition ordering. The counterexample is a response that -arrives before a request-written marker, an absent/ambiguous marker, or a -second poll that returns a message after withdrawal; each case is recorded as +SDK's ACK-before-acquisition ordering. Counterexamples include a response that +arrives before a request-written marker, an absent/ambiguous marker, a missing +controlled old message, a replacement/disappearing runner, unknown or +contradictory counters, a duplicate/wrong callback, or a second poll that +returns a message after withdrawal. Each is recorded as inconclusive/quarantined and never treated as proof of a drain barrier. The transport hook records `WroteRequest` and holds the first response body @@ -33,39 +35,75 @@ retains uncertainty. Neither outcome authorizes deletion, replay, re-acquisition or runner removal. The prerequisite remains one already-idle, registered, owned canary runner -with zero available/acquired/assigned/running/busy work. No idle-worker -pre-provision seam was added; if a future live run needs one, it requires a -separate independent design review and explicit maintainer authorization. +with zero available/acquired/assigned/running/busy work. The complete bounded +prerequisite snapshot is journaled before the listener starts; a rejected or +ambiguous snapshot writes a fixed quarantine marker, so later zero statistics +cannot authorize cleanup. No idle-worker pre-provision seam was added; if a +future live run needs one, it requires a separate independent design review +and explicit maintainer authorization. ## TDD and verification -Meaningful behavioral red was captured before the implementation existed: +The first implementation's historical red was: ```text cd experiments/g01-scaleset go test ./livecanary -run 'TestDrainListener' -count=1 ``` -The module failed to compile because the drain hook, listener runner, -boundary/category constants and experiment seam were undefined. After the -minimal implementation, the same behavior was green, followed by failure and -boundary coverage for missing request-written evidence, negative counters, -identity changes, duplicate ordering, response-before-write observations, -idle prerequisites, replay uncertainty and missing verification authority. +It failed to compile because the drain hook, listener runner, +boundary/category constants and experiment seam were undefined. This is +retained as chronology only and is not claimed as meaningful behavioral TDD. +The meaningful behavioral red was reproduced against the immutable base +`cf67d4a`: `TestIssue71DrainAuthorityIsBehaviorallyAvailable` failed because +the base rejected the new `drain` phase with `approval rejected`. The equivalent +green test is `TestDrainPhaseAuthorityIsAccepted` on fix commit +`82d0d7d`. + +Before the correction commit, a temporary test-only checkout pinned to +`6e954cf` reproduced all five independent findings with executable assertions: +poll reservation omitted, rejected idle state not fencing replay, wrong ACK +reaching the inner effect, no-message accepted as observed, and cancellation +missing an explicit marker. All five assertions failed on that pre-correction +head; the focused tests below pass on `82d0d7d`. + +## Independent correction matrix + +| Finding | Correction and evidence | +|---|---| +| Codex P1 status-only response close (`r3965776832`) | `drainHeldBody.Close` remains release-gated; `TestDrainHeldBodyHoldsCloseUntilRelease` passes (fix `f44f5f9`). | +| Codex P1 cancellation join (`r3965776826`) | all listener context exits release and join the run goroutine; `TestDrainRejectsEffectsAfterCancellationAndRecordsMarker` passes (fixes `f44f5f9`, `82d0d7d`). | +| Codex P1 drain route unreachable (`r3965776815`) | `Run` routes `drain` before generic owned/statistics quarantine; `TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine` passes (fix `f44f5f9`). | +| Codex P2 paired verification (`r3965957331`) | drain is admitted as a verification phase in paired approval/preparation; paired tests pass (fix `945371e`). | +| Codex P1 runner continuity (`r3966125441`) | observed requires exact before/after runner tuple; runner mutation is rejected (fix `6e954cf`). | +| Security F1 / Protocol F1 poll reservation | `observe-poll` result journals one verified request ID plus fixed work before ACK; replay sets reservation/work fences. `TestDrainPollJournalsReservationBeforeACK` passes (`82d0d7d`). | +| Security F2 rejected idle prerequisite | complete bounded `DrainSnapshot` plus `prerequisite-failed` marker is retained; replay remains uncertain. `TestDrainRejectedIdlePrerequisiteRetainsFence` passes (`82d0d7d`). | +| Security F3 / Protocol F2 runner and nested set identity | observed requires exact runner continuity; session-open validates nested set ID/name/group/label/update fence. Pinned SDK drain integration and mutation tests pass (`6e954cf`, `82d0d7d`). | +| Security F4 / Protocol F1 no-message | observed requires a present old controlled message, known counters, successful ACK and acquisition; 202/no-message is inconclusive. `TestDrainListenerNoMessageIsInconclusive` passes (`82d0d7d`). | +| Security F5 / Protocol F5 counters and integration | known counters require nonnegative internally consistent busy/idle partition; unknown values cannot masquerade as zero. `TestDrainRequiresKnownConsistentStatisticsAndControlledMessage` and `TestDriverDrainThroughPinnedSDKAndPollHook` pass (`82d0d7d`). | +| Protocol F3 cancellation/WithoutCancel | phase context is bound through the wrapper, checked before every ACK/acquisition/poll effect, and cancellation/deadline/quarantine writes a fixed marker even when close/after inspect fails. Focused race test passes (`82d0d7d`). | +| Protocol F4 duplicate/wrong callbacks | exact phase, identity, ACK-before-acquire and one-shot state are validated before inner effects; wrong/duplicate callbacks make zero additional inner calls. `TestDrainRejectsDuplicateOrWrongEffectsBeforeInnerCall` passes (`82d0d7d`). | + +The independent design review remains respected: the transport marker is +client-side only, `server_receipt` is `unproven`, the real high-level pinned +listener is retained, the hook is bounded to one old and one next poll, and no +pre-provision, recovery, replay, `RemoveRunner` or live operation was added. Verified locally with the pinned `github.com/actions/scaleset v0.4.0` module: ```text cd experiments/g01-scaleset go test ./livecanary -run 'TestDrain|TestCredentialAttestationMismatchAndExpiredTokenRejected' -count=1 -go test -race ./livecanary -run 'TestDrainListener' -count=1 +go test -race ./livecanary -run 'TestDrain|TestDriverDrainThroughPinnedSDK' -count=1 cd ../g02-auth go test ./... -run 'TestBrokerFinitePhases|TestBrokerControllerApproval' -count=1 ``` -All commands above passed. The listener test uses an `httptest` synthetic -transport and the real released listener; it is offline protocol evidence, -not live qualification. No credentials, runner/session/JIT operation, +All commands above passed, as did `go test ./...` in both experiment modules. +The listener tests use `httptest` synthetic transports and the real released +listener; the pinned-SDK test composes `OpenDrainSession` with the actual +`MessageSessionClient` offline. These are offline protocol evidence, not live +qualification. No credentials, runner/session/JIT operation, workflow operation, app/keychain/launchd/Docker/Lima mutation or cleanup was performed. From f482d249e6e5eca7bd03ce55cdaf3b6cde7a671d Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 18:07:40 +0900 Subject: [PATCH 07/63] fix(g01): retain ambiguous drain sessions --- docs/evidence/g01-idle-drain.md | 3 ++ .../g01-scaleset/livecanary/drain_driver.go | 38 +++++++++++--- .../g01-scaleset/livecanary/drain_test.go | 52 ++++++++++++++++--- experiments/g01-scaleset/livecanary/driver.go | 5 ++ 4 files changed, 84 insertions(+), 14 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index c2b4f06..f05f4eb 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -76,6 +76,9 @@ head; the focused tests below pass on `82d0d7d`. | Codex P1 drain route unreachable (`r3965776815`) | `Run` routes `drain` before generic owned/statistics quarantine; `TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine` passes (fix `f44f5f9`). | | Codex P2 paired verification (`r3965957331`) | drain is admitted as a verification phase in paired approval/preparation; paired tests pass (fix `945371e`). | | Codex P1 runner continuity (`r3966125441`) | observed requires exact before/after runner tuple; runner mutation is rejected (fix `6e954cf`). | +| Codex P1 embedded session set (`r3966362999`) | session-open now requires exact nested set identity, update fence, labels, and statistics equal to the idle before snapshot. `TestDrainRejectsEmbeddedSessionStatisticsMismatch` passes (fix `82d0d7d`). | +| Codex P1 ambiguous session retention (`r3966363009`) | non-successful listener outcomes retain the live session, write a quarantine marker, and never invoke session close; `TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine` asserts zero close calls (fix `82d0d7d`). | +| Codex P1 phase crash fence (`r3966362990`) | replay marks a durable `drain` phase uncertain before polling, so a crash before the final observation cannot authorize cleanup. `TestDrainPhaseStartRetainsCrashUncertainty` passes (fix `82d0d7d`). | | Security F1 / Protocol F1 poll reservation | `observe-poll` result journals one verified request ID plus fixed work before ACK; replay sets reservation/work fences. `TestDrainPollJournalsReservationBeforeACK` passes (`82d0d7d`). | | Security F2 rejected idle prerequisite | complete bounded `DrainSnapshot` plus `prerequisite-failed` marker is retained; replay remains uncertain. `TestDrainRejectedIdlePrerequisiteRetainsFence` passes (`82d0d7d`). | | Security F3 / Protocol F2 runner and nested set identity | observed requires exact runner continuity; session-open validates nested set ID/name/group/label/update fence. Pinned SDK drain integration and mutation tests pass (`6e954cf`, `82d0d7d`). | diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index be9ccbf..ef772dd 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -216,11 +216,12 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return Event{}, ErrRemote } current := session.Session() - if current.SessionID == [16]byte{} || current.OwnerName != d.Approval.setName() || current.RunnerScaleSet == nil || current.RunnerScaleSet.ID != setID || current.RunnerScaleSet.Name != d.Approval.setName() || current.RunnerScaleSet.RunnerGroupID != d.Approval.RunnerGroupID || !current.RunnerScaleSet.RunnerSetting.DisableUpdate || !slices.ContainsFunc(current.RunnerScaleSet.Labels, func(label scaleset.Label) bool { return label.Name == d.Approval.setName() }) || current.Statistics == nil { + if current.SessionID == [16]byte{} || current.OwnerName != d.Approval.setName() || current.RunnerScaleSet == nil || current.RunnerScaleSet.ID != setID || current.RunnerScaleSet.Name != d.Approval.setName() || current.RunnerScaleSet.RunnerGroupID != d.Approval.RunnerGroupID || !current.RunnerScaleSet.RunnerSetting.DisableUpdate || !slices.ContainsFunc(current.RunnerScaleSet.Labels, func(label scaleset.Label) bool { return label.Name == d.Approval.setName() }) || current.Statistics == nil || current.RunnerScaleSet.Statistics == nil { return Event{}, ErrQuarantine } stats, err := newDrainStatistics(current.Statistics) - if err != nil || stats != before.Statistics { + embeddedStats, embeddedErr := newDrainStatistics(current.RunnerScaleSet.Statistics) + if err != nil || embeddedErr != nil || stats != before.Statistics || embeddedStats != before.Statistics { return Event{}, ErrQuarantine } sessionID = current.SessionID.String() @@ -234,12 +235,6 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID} obs, runErr := runDrainListener(ctx, journaled, setID, hook) - closeErr := d.effect(ctx, "session-close", nil, func(call context.Context) (Event, error) { - if err := session.Close(call); err != nil { - return Event{}, err - } - return Event{SessionID: sessionID}, nil - }) if ctx.Err() != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, runErr)); markerErr != nil { return markerErr @@ -249,6 +244,33 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return ErrQuarantine } + if runErr != nil { + // An unknown ACK/acquisition or a timing-miss leaves the session as a + // live reservation. Closing it can requeue or otherwise change the + // remote state before an operator can inspect the journal. + if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, runErr)); markerErr != nil { + return markerErr + } + after, afterErr := d.drainSnapshot(ctx, setID, "after") + if afterErr != nil { + if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, afterErr)); markerErr != nil { + return markerErr + } + return afterErr + } + obs.Before, obs.After = before, after + obs.Outcome = drainOutcomeInconclusive + if recordErr := d.record(Event{Kind: "observation", Operation: "drain", Drain: &obs}); recordErr != nil { + return recordErr + } + return runErr + } + closeErr := d.effect(ctx, "session-close", nil, func(call context.Context) (Event, error) { + if err := session.Close(call); err != nil { + return Event{}, err + } + return Event{SessionID: sessionID}, nil + }) if closeErr != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, closeErr)); markerErr != nil { return markerErr diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 564cca7..8ebb861 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -277,6 +277,13 @@ func TestDrainPhaseAuthorityIsAccepted(t *testing.T) { } } +func TestDrainPhaseStartRetainsCrashUncertainty(t *testing.T) { + state := replay([]Event{{Kind: "phase", Operation: "drain"}}) + if !state.uncertain { + t.Fatal("drain phase without final observation did not retain uncertainty") + } +} + func TestDrainPollJournalsReservationBeforeACK(t *testing.T) { a := approval() j := &memoryJournal{} @@ -333,6 +340,24 @@ func TestDrainRejectedIdlePrerequisiteRetainsFence(t *testing.T) { t.Fatal("rejected prerequisite did not persist a bounded marker") } +func TestDrainRejectsEmbeddedSessionStatisticsMismatch(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + j := &memoryJournal{events: []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + }} + api := &drainDriverAPI{fakeAPI: &fakeAPI{}, approval: a, sessionStats: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1, TotalAssignedJobs: 1}} + d := Driver{Approval: a, Journal: j, API: api} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("embedded session mismatch = %v, want quarantine", err) + } + if replay(j.Events()).uncertain == false { + t.Fatal("embedded session mismatch did not retain uncertainty") + } +} + func TestDrainRejectsEffectsAfterCancellationAndRecordsMarker(t *testing.T) { a := approval() a.Phases = append(a.Phases, "drain") @@ -450,6 +475,9 @@ func TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine(t *testing.T) { if api.getScaleSetCalls != 2 { t.Fatalf("owned set reads = %d, want before/after drain reads", api.getScaleSetCalls) } + if api.closeCalls != 0 { + t.Fatalf("ambiguous drain closed session %d times", api.closeCalls) + } var observed bool for _, event := range j.events { if event.Kind == "observation" && event.Operation == "drain" { @@ -499,6 +527,8 @@ type drainDriverAPI struct { snapshotStats *scaleset.RunnerScaleSetStatistic blocking bool opened chan struct{} + closeCalls int + sessionStats *scaleset.RunnerScaleSetStatistic } func (a *drainDriverAPI) GetScaleSet(context.Context, int) (*scaleset.RunnerScaleSet, error) { @@ -527,12 +557,16 @@ func (a *drainDriverAPI) OpenDrainSession(_ context.Context, _ int, _ string, ho hook.mu.Lock() hook.target = "http://fixture.invalid/queue" hook.mu.Unlock() - return &drainBlockingSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: hook.target, RunnerScaleSet: &scaleset.RunnerScaleSet{ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, Labels: []scaleset.Label{{Name: a.approval.setName()}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}}, nil + return &drainBlockingSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: hook.target, RunnerScaleSet: &scaleset.RunnerScaleSet{ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, Labels: []scaleset.Label{{Name: a.approval.setName()}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}}, nil } hook.mu.Lock() hook.target = "http://fixture.invalid/queue" hook.mu.Unlock() - return &drainSyntheticSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: "http://fixture.invalid/queue", RunnerScaleSet: &scaleset.RunnerScaleSet{ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, Labels: []scaleset.Label{{Name: a.approval.setName()}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}}, nil + stats := a.sessionStats + if stats == nil { + stats = &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1} + } + return &drainSyntheticSession{initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: a.approval.setName(), MessageQueueURL: "http://fixture.invalid/queue", RunnerScaleSet: &scaleset.RunnerScaleSet{ID: 7, Name: a.approval.setName(), RunnerGroupID: a.approval.RunnerGroupID, Labels: []scaleset.Label{{Name: a.approval.setName()}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, Statistics: stats}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}, closeCalls: &a.closeCalls}, nil } type drainGuardSession struct { @@ -608,13 +642,19 @@ func (t *drainNoTraceTransport) RoundTrip(req *http.Request) (*http.Response, er } type drainSyntheticSession struct { - client *http.Client - initial scaleset.RunnerScaleSetSession - order *[]string + client *http.Client + initial scaleset.RunnerScaleSetSession + order *[]string + closeCalls *int } func (s *drainSyntheticSession) Session() scaleset.RunnerScaleSetSession { return s.initial } -func (s *drainSyntheticSession) Close(context.Context) error { return nil } +func (s *drainSyntheticSession) Close(context.Context) error { + if s.closeCalls != nil { + (*s.closeCalls)++ + } + return nil +} func (s *drainSyntheticSession) GetMessage(ctx context.Context, _, capacity int) (*scaleset.RunnerScaleSetMessage, error) { if s.client == nil { return nil, errors.New("synthetic transport unavailable") diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index 940043d..d999153 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -110,6 +110,11 @@ func replay(events []Event) state { s.reserved, s.uncertain, s.workObserved = true, true, true case "phase": s.phaseSeen[e.Operation] = true + if e.Operation == "drain" { + // The phase record is durable intent to observe a live session. + // A crash before its final observation must not reopen cleanup. + s.uncertain = true + } case "inventory": s.inventory = e.Digest case "observation": From 02ba76466da75ffb57276b5b6b66512441ea3520 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 18:16:03 +0900 Subject: [PATCH 08/63] chore(g01): format drain SDK adapter --- experiments/g01-scaleset/livecanary/sdk.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 10632b5..c17cfba 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -51,12 +51,12 @@ func (c Credentials) validate(a Approval, now time.Time) error { } type SDKAPI struct { - client *scaleset.Client - rest *http.Client - baseURL string - approval Approval - credentials Credentials - options []scaleset.HTTPOption + client *scaleset.Client + rest *http.Client + baseURL string + approval Approval + credentials Credentials + options []scaleset.HTTPOption // drainClientFactory is nil in production. Tests use it only to bind the // pinned SDK client to an offline loopback transport while still exercising // OpenDrainSession and MessageSessionClient together. From 21fe1f846d372699b10525bf4916e1d83004989b Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 18:51:18 +0900 Subject: [PATCH 09/63] fix(g01): fence unknown drain samples and cancellation --- docs/evidence/g01-idle-drain.md | 58 +++-- experiments/g01-scaleset/livecanary/drain.go | 230 +++++++++++++++++- .../g01-scaleset/livecanary/drain_driver.go | 3 + .../g01-scaleset/livecanary/drain_test.go | 127 +++++++++- experiments/g01-scaleset/livecanary/driver.go | 24 ++ .../livecanary/sdk_integration_test.go | 6 +- 6 files changed, 423 insertions(+), 25 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index f05f4eb..d53d336 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -54,18 +54,35 @@ go test ./livecanary -run 'TestDrainListener' -count=1 It failed to compile because the drain hook, listener runner, boundary/category constants and experiment seam were undefined. This is retained as chronology only and is not claimed as meaningful behavioral TDD. -The meaningful behavioral red was reproduced against the immutable base -`cf67d4a`: `TestIssue71DrainAuthorityIsBehaviorallyAvailable` failed because -the base rejected the new `drain` phase with `approval rejected`. The equivalent -green test is `TestDrainPhaseAuthorityIsAccepted` on fix commit +A meaningful behavioral counterexample was later reproduced retrospectively +against immutable base `cf67d4a`: `TestIssue71DrainAuthorityIsBehaviorallyAvailable` +failed because the base rejected the new `drain` phase with `approval rejected`. +That is independently inspectable base behavior, not a pre-implementation +run; the original pre-implementation meaningful-red chronology is unavailable. +The equivalent green test is `TestDrainPhaseAuthorityIsAccepted` on fix commit `82d0d7d`. -Before the correction commit, a temporary test-only checkout pinned to -`6e954cf` reproduced all five independent findings with executable assertions: -poll reservation omitted, rejected idle state not fencing replay, wrong ACK -reaching the inner effect, no-message accepted as observed, and cancellation -missing an explicit marker. All five assertions failed on that pre-correction -head; the focused tests below pass on `82d0d7d`. +Retrospectively, after the correction commit, a temporary test-only checkout +pinned to `6e954cf` reproduced all five independent findings with executable +assertions: poll reservation omitted, rejected idle state not fencing replay, +wrong ACK reaching the inner effect, no-message accepted as observed, and +cancellation missing an explicit marker. Those failures are defect evidence, +not a claim that the tests preceded implementation; the focused regressions +below pass on the current correction head. + +The newly queued f482 defects were first reproduced behaviorally on the +current f482 worktree with the added regressions: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestDrain(RequiresKnownConsistentStatisticsAndControlledMessage|CancellationAfterIntentRejectsEffect|CancellationBeforeSnapshotRecordsMarker)$' -count=1 -v +FAIL: unknown next-poll statistics were accepted; cancellation after ACK intent reached the inner ACK; before-snapshot cancellation had no marker. +``` + +The minimal correction then made those assertions pass, along with the +bounded adapter field-presence test; the cancellation fence is deliberately +not described as atomic against a remote call that was already issued or +accepted. ## Independent correction matrix @@ -82,10 +99,12 @@ head; the focused tests below pass on `82d0d7d`. | Security F1 / Protocol F1 poll reservation | `observe-poll` result journals one verified request ID plus fixed work before ACK; replay sets reservation/work fences. `TestDrainPollJournalsReservationBeforeACK` passes (`82d0d7d`). | | Security F2 rejected idle prerequisite | complete bounded `DrainSnapshot` plus `prerequisite-failed` marker is retained; replay remains uncertain. `TestDrainRejectedIdlePrerequisiteRetainsFence` passes (`82d0d7d`). | | Security F3 / Protocol F2 runner and nested set identity | observed requires exact runner continuity; session-open validates nested set ID/name/group/label/update fence. Pinned SDK drain integration and mutation tests pass (`6e954cf`, `82d0d7d`). | -| Security F4 / Protocol F1 no-message | observed requires a present old controlled message, known counters, successful ACK and acquisition; 202/no-message is inconclusive. `TestDrainListenerNoMessageIsInconclusive` passes (`82d0d7d`). | +| Security F4 / Protocol F1 no-message and field presence | observed requires a present old controlled message, complete known counters on both polls, successful ACK and acquisition; bodyless 202/no-message and empty/missing statistics objects remain inconclusive. The bounded adapter retains only field presence/scalars ephemerally; `TestDrainListenerNoMessageIsInconclusive`, `TestDrainRequiresKnownConsistentStatisticsAndControlledMessage`, and `TestDrainPollHookPreservesStatisticsFieldPresence` pass on the correction head. | | Security F5 / Protocol F5 counters and integration | known counters require nonnegative internally consistent busy/idle partition; unknown values cannot masquerade as zero. `TestDrainRequiresKnownConsistentStatisticsAndControlledMessage` and `TestDriverDrainThroughPinnedSDKAndPollHook` pass (`82d0d7d`). | -| Protocol F3 cancellation/WithoutCancel | phase context is bound through the wrapper, checked before every ACK/acquisition/poll effect, and cancellation/deadline/quarantine writes a fixed marker even when close/after inspect fails. Focused race test passes (`82d0d7d`). | +| Protocol F3 cancellation/WithoutCancel | phase context is bound through the wrapper, checked before every ACK/acquisition/poll effect, and cancellation/deadline/quarantine writes a fixed marker even when close/after inspect fails. `TestDrainRejectsEffectsAfterCancellationAndRecordsMarker` passes (`82d0d7d`). | | Protocol F4 duplicate/wrong callbacks | exact phase, identity, ACK-before-acquire and one-shot state are validated before inner effects; wrong/duplicate callbacks make zero additional inner calls. `TestDrainRejectsDuplicateOrWrongEffectsBeforeInnerCall` passes (`82d0d7d`). | +| Protocol F6 cancellation race and early snapshot | durable intent is followed by a cancellation fence immediately before the bounded SDK call; a canceled before-snapshot path records its fixed marker. The fence does not claim to revoke bytes already accepted by a remote service. `TestDrainCancellationAfterIntentRejectsEffect` and `TestDrainCancellationBeforeSnapshotRecordsMarker` pass on the correction head. | +| Protocol F9 TDD chronology | the compile-only red and retrospective base/repro executions are now labeled candidly; no later archive reproduction is represented as pre-implementation evidence. | The independent design review remains respected: the transport marker is client-side only, `server_receipt` is `unproven`, the real high-level pinned @@ -96,13 +115,20 @@ Verified locally with the pinned `github.com/actions/scaleset v0.4.0` module: ```text cd experiments/g01-scaleset -go test ./livecanary -run 'TestDrain|TestCredentialAttestationMismatchAndExpiredTokenRejected' -count=1 -go test -race ./livecanary -run 'TestDrain|TestDriverDrainThroughPinnedSDK' -count=1 +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestDrain|TestCredentialAttestationMismatchAndExpiredTokenRejected' -count=1 +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestDrain|TestDriverDrainThroughPinnedSDK' -count=1 +GOTOOLCHAIN=go1.26.8 go vet ./livecanary cd ../g02-auth -go test ./... -run 'TestBrokerFinitePhases|TestBrokerControllerApproval' -count=1 +GOTOOLCHAIN=go1.26.8 go test ./... -run 'TestBrokerFinitePhases|TestBrokerControllerApproval' -count=1 +cd ../.. +git diff --check +bash scripts/gofmt.sh check +bash scripts/check-offline-experiments.sh ``` -All commands above passed, as did `go test ./...` in both experiment modules. +All commands above passed, as did `GOTOOLCHAIN=go1.26.8 go test ./...` in both +experiment modules and the focused/race drain suites. The offline script +reported `offline experiment checks passed: 2 module(s)`. The listener tests use `httptest` synthetic transports and the real released listener; the pinned-SDK test composes `OpenDrainSession` with the actual `MessageSessionClient` offline. These are offline protocol evidence, not live diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 1b6b8de..e306f7c 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -1,8 +1,11 @@ package livecanary import ( + "bytes" "context" + "encoding/json" "errors" + "io" "net/http" "net/http/httptrace" "net/url" @@ -83,6 +86,58 @@ type drainPollObservation struct { Acquisition string `json:"acquisition"` } +var drainStatisticsFields = [...]string{ + "totalAvailableJobs", + "totalAcquiredJobs", + "totalAssignedJobs", + "totalRunningJobs", + "totalRegisteredRunners", + "totalBusyRunners", + "totalIdleRunners", +} + +// drainStatisticsFromBody preserves only field presence and bounded scalar +// values at the SDK adapter boundary. The pinned SDK decodes an empty JSON +// object into a non-nil all-zero struct, so a pointer alone is not evidence +// that the service supplied a complete statistics sample. +func drainStatisticsFromBody(body []byte) (drainStatistics, bool) { + var envelope map[string]json.RawMessage + if json.Unmarshal(body, &envelope) != nil { + return drainStatistics{}, false + } + raw, ok := envelope["statistics"] + if !ok || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { + return drainStatistics{}, false + } + var fields map[string]json.RawMessage + if json.Unmarshal(raw, &fields) != nil { + return drainStatistics{}, false + } + if len(fields) < len(drainStatisticsFields) { + return drainStatistics{}, false + } + values := make([]int, len(drainStatisticsFields)) + for i, field := range drainStatisticsFields { + rawValue, ok := fields[field] + if !ok || bytes.Equal(bytes.TrimSpace(rawValue), []byte("null")) || json.Unmarshal(rawValue, &values[i]) != nil { + return drainStatistics{}, false + } + } + stats := drainStatistics{ + Available: values[0], + Acquired: values[1], + Assigned: values[2], + Running: values[3], + Registered: values[4], + Busy: values[5], + Idle: values[6], + } + if !validKnownDrainStatistics(stats) { + return drainStatistics{}, false + } + return stats, true +} + type drainSetIdentity struct { ID int `json:"id"` Name string `json:"name"` @@ -174,7 +229,7 @@ func validDrainObservation(o *drainObservation) bool { seen[item] = true } if o.Outcome == drainOutcomeObserved { - if o.Poll.Message != drainMessagePresent || !o.Poll.StatsKnown || o.NextPoll.Message != drainMessageAbsent || o.NextPoll.ACK != drainResponseNotAttempted || o.NextPoll.Acquisition != drainResponseNotAttempted { + if o.Poll.Message != drainMessagePresent || !o.Poll.StatsKnown || o.Poll.Statistics == (drainStatistics{}) || o.NextPoll.Message != drainMessageAbsent || !o.NextPoll.StatsKnown || o.NextPoll.ACK != drainResponseNotAttempted || o.NextPoll.Acquisition != drainResponseNotAttempted { return false } if o.Poll.ACK != drainResponseSucceeded || o.Poll.Acquisition != drainResponseSucceeded { @@ -214,6 +269,8 @@ type drainPollHook struct { responseHeld bool invalid bool pollAttempts int + statistics [3]drainStatistics + statisticsKnown [3]bool onRequestWritten func() wroteOnce sync.Once responseOnce sync.Once @@ -249,6 +306,7 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { } h.mu.Lock() h.pollAttempts++ + attempt := h.pollAttempts first := h.pollAttempts == 1 h.mu.Unlock() if !first { @@ -261,7 +319,27 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { if !second { return nil, ErrQuarantine } - return h.inner.RoundTrip(req) + response, err := h.inner.RoundTrip(req) + if err != nil { + return nil, err + } + if response == nil { + return nil, ErrQuarantine + } + if response.Body == nil { + response.Body = http.NoBody + } + response.Body = &drainObservedBody{ + source: response.Body, + status: response.StatusCode, + onComplete: func(stats drainStatistics, known bool) { + h.mu.Lock() + h.statistics[attempt] = stats + h.statisticsKnown[attempt] = known + h.mu.Unlock() + }, + } + return response, nil } trace := &httptrace.ClientTrace{WroteRequest: func(info httptrace.WroteRequestInfo) { @@ -306,11 +384,126 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { if response.Body == nil { response.Body = http.NoBody } - response.Body = &drainHeldBody{source: response.Body, release: h.release} + observedBody := &drainObservedBody{ + source: response.Body, + status: response.StatusCode, + onComplete: func(stats drainStatistics, known bool) { + h.mu.Lock() + if attempt >= 1 && attempt < len(h.statistics) { + h.statistics[attempt] = stats + h.statisticsKnown[attempt] = known + } + h.mu.Unlock() + }, + } + response.Body = &drainHeldBody{source: observedBody, release: h.release} h.responseOnce.Do(func() { close(h.response) }) return response, nil } +func (h *drainPollHook) pollStatistics(index int) (drainStatistics, bool) { + if h == nil || index < 1 || index >= len(h.statistics) { + return drainStatistics{}, false + } + h.mu.Lock() + defer h.mu.Unlock() + return h.statistics[index], h.statisticsKnown[index] +} + +// drainObservedBody forwards response bytes unchanged while retaining a +// bounded, ephemeral copy solely to establish statistics field presence. It +// never persists, logs or rewrites the response payload. +type drainObservedBody struct { + source io.ReadCloser + status int + data []byte + over bool + mu sync.Mutex + finished bool + onComplete func(drainStatistics, bool) +} + +func (b *drainObservedBody) capture(data []byte) { + b.mu.Lock() + defer b.mu.Unlock() + remaining := int(responseBodyLimit) - len(b.data) + if remaining <= 0 { + b.over = true + return + } + if len(data) > remaining { + b.data = append(b.data, data[:remaining]...) + b.over = true + return + } + b.data = append(b.data, data...) +} + +func (b *drainObservedBody) finish() { + b.mu.Lock() + if b.finished { + b.mu.Unlock() + return + } + b.finished = true + data := append([]byte(nil), b.data...) + over := b.over + b.data = nil + b.mu.Unlock() + + stats, known := drainStatisticsFromBody(data) + if b.status != http.StatusOK && b.status != http.StatusAccepted { + known = false + stats = drainStatistics{} + } + if over { + known = false + stats = drainStatistics{} + } + if b.onComplete != nil { + b.onComplete(stats, known) + } +} + +func (b *drainObservedBody) Read(p []byte) (int, error) { + n, err := b.source.Read(p) + if n > 0 { + b.capture(p[:n]) + } + if err != nil { + b.finish() + } + return n, err +} + +func (b *drainObservedBody) Close() error { + b.mu.Lock() + finished := b.finished + remaining := int(responseBodyLimit) - len(b.data) + b.mu.Unlock() + if !finished { + if remaining < 0 { + remaining = 0 + } + data, err := io.ReadAll(io.LimitReader(b.source, int64(remaining)+1)) + if len(data) > remaining { + b.mu.Lock() + b.over = true + b.mu.Unlock() + } + if len(data) > 0 { + b.capture(data) + } + if err != nil { + b.mu.Lock() + b.over = true + b.mu.Unlock() + } + b.finish() + } + return b.source.Close() +} + func (h *drainPollHook) releaseResponse() { if h == nil { return @@ -356,6 +549,7 @@ type drainClient struct { inner listener.Client validate func(context.Context, *scaleset.RunnerScaleSetMessage) error obs *drainObservation + hook *drainPollHook phaseCtx context.Context reject func(string) error mu sync.Mutex @@ -415,20 +609,42 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal c.mu.Unlock() message, err := c.inner.GetMessage(ctx, last, capacity) + stats, statsKnown := drainStatistics{}, false + if c.hook == nil { + if message != nil { + var statsErr error + stats, statsErr = newDrainStatistics(message.Statistics) + statsKnown = statsErr == nil + } + } else { + stats, statsKnown = c.hook.pollStatistics(index) + } if err != nil { c.mu.Lock() c.obs.poll(index).Message = drainMessageUnknown + c.obs.poll(index).Statistics = drainStatistics{} + c.obs.poll(index).StatsKnown = false c.mu.Unlock() return nil, err } if message == nil { c.mu.Lock() c.obs.poll(index).Message = drainMessageAbsent + c.obs.poll(index).Statistics = stats + c.obs.poll(index).StatsKnown = statsKnown c.mu.Unlock() return nil, nil } - stats, statsErr := newDrainStatistics(message.Statistics) - if statsErr != nil || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { + if !statsKnown || message.Statistics == nil { + c.mu.Lock() + c.obs.poll(index).Message = drainMessageUnknown + c.obs.poll(index).Statistics = drainStatistics{} + c.obs.poll(index).StatsKnown = false + c.mu.Unlock() + return nil, ErrQuarantine + } + messageStats, statsErr := newDrainStatistics(message.Statistics) + if statsErr != nil || (c.hook != nil && stats != messageStats) || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { c.mu.Lock() c.obs.poll(index).Message = drainMessageUnknown c.mu.Unlock() @@ -558,7 +774,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho obs.Poll.Statistics = stats obs.Poll.StatsKnown = true } - c := &drainClient{inner: client, obs: &obs} + c := &drainClient{inner: client, obs: &obs, hook: hook} if rejecter, ok := client.(interface{ reject(string) error }); ok { c.reject = rejecter.reject } @@ -646,7 +862,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho if obs.Boundary == "" { obs.Boundary, obs.ResponseHeld, _ = hook.boundary() } - if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld && obs.Poll.Message == drainMessagePresent && obs.Poll.StatsKnown && obs.Poll.ACK == drainResponseSucceeded && obs.Poll.Acquisition == drainResponseSucceeded && obs.NextPoll.Message == drainMessageAbsent { + if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld && obs.Poll.Message == drainMessagePresent && obs.Poll.StatsKnown && obs.Poll.Statistics != (drainStatistics{}) && obs.Poll.ACK == drainResponseSucceeded && obs.Poll.Acquisition == drainResponseSucceeded && obs.NextPoll.Message == drainMessageAbsent && obs.NextPoll.StatsKnown { obs.Outcome = drainOutcomeObserved return obs, nil } diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index ef772dd..96b72d3 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -198,6 +198,9 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } before, err := d.drainSnapshot(ctx, setID, "before") if err != nil { + if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, err)); markerErr != nil { + return markerErr + } return err } if !validDrainIdlePrerequisite(before) { diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 8ebb861..0b59090 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -50,7 +50,9 @@ func TestDrainListenerWithdrawsWhilePollResponseIsHeld(t *testing.T) { mu.Unlock() return } + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) + _, _ = io.WriteString(w, `{"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`) mu.Unlock() return } @@ -154,7 +156,7 @@ func TestDrainListenerNoMessageIsInconclusive(t *testing.T) { client.Transport = hook session.client = &client observation, err := runDrainListener(context.Background(), session, 7, hook) - if !errors.Is(err, ErrNoMessage) || observation.Outcome != drainOutcomeInconclusive || observation.Poll.Message != drainMessageAbsent { + if !errors.Is(err, ErrNoMessage) || observation.Outcome != drainOutcomeInconclusive || observation.Poll.Message != drainMessageAbsent || observation.Poll.StatsKnown || observation.NextPoll.StatsKnown { t.Fatalf("no-message drain = %+v err=%v, want inconclusive", observation, err) } } @@ -456,6 +458,123 @@ func TestDrainRequiresKnownConsistentStatisticsAndControlledMessage(t *testing.T if validDrainObservation(&noMessage) { t.Fatal("observed drain with no controlled old message accepted") } + unknownNext := observation + unknownNext.NextPoll.StatsKnown = false + unknownNext.NextPoll.Statistics = drainStatistics{} + if validDrainObservation(&unknownNext) { + t.Fatal("observed drain with unknown next-poll statistics accepted") + } + emptyStatistics := observation + emptyStatistics.Poll.Statistics = drainStatistics{} + if validDrainObservation(&emptyStatistics) { + t.Fatal("observed drain with empty controlled-poll statistics accepted") + } +} + +type drainCancelOnIntentJournal struct { + *memoryJournal + cancel context.CancelFunc +} + +func (j *drainCancelOnIntentJournal) Append(event Event) error { + if event.Kind == "intent" && event.Operation == "ack" { + j.cancel() + } + return j.memoryJournal.Append(event) +} + +func TestDrainCancellationAfterIntentRejectsEffect(t *testing.T) { + a := approval() + ctx, cancel := context.WithCancel(context.Background()) + j := &drainCancelOnIntentJournal{memoryJournal: &memoryJournal{}, cancel: cancel} + f := &fakeAPI{session: &fakeSession{session: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), Statistics: &scaleset.RunnerScaleSetStatistic{}}}} + d := &Driver{Approval: a, Journal: j, API: f} + c := &journaledDrainClient{d: d, inner: f.session, sessionID: "session"} + c.bindDrainContext(ctx) + err := c.DeleteMessage(context.WithoutCancel(ctx), 7) + if !errors.Is(err, ErrQuarantine) || f.session.ack != 0 { + t.Fatalf("cancellation race effect = err %v ack %d; want quarantine and zero calls", err, f.session.ack) + } +} + +type drainCancelBeforeSnapshotAPI struct { + *drainDriverAPI + cancel context.CancelFunc +} + +func (a *drainCancelBeforeSnapshotAPI) Preflight(context.Context, Approval) error { + a.cancel() + return nil +} + +func (*drainCancelBeforeSnapshotAPI) GetScaleSet(context.Context, int) (*scaleset.RunnerScaleSet, error) { + return nil, context.Canceled +} + +func TestDrainCancellationBeforeSnapshotRecordsMarker(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + j := &memoryJournal{events: []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + }} + ctx, cancel := context.WithCancel(context.Background()) + api := &drainCancelBeforeSnapshotAPI{ + drainDriverAPI: &drainDriverAPI{fakeAPI: &fakeAPI{}, approval: a}, + cancel: cancel, + } + d := &Driver{Approval: a, Journal: j, API: api} + if err := d.Run(ctx, "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("before-snapshot cancellation = %v, want quarantine", err) + } + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain-marker" && event.DrainMarker == drainMarkerCancelled { + return + } + } + t.Fatal("before-snapshot cancellation did not persist a cancellation marker") +} + +func TestDrainPollHookPreservesStatisticsFieldPresence(t *testing.T) { + full := `{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[]"}` + empty := `{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{},"body":"[]"}` + missing := `{"messageId":7,"messageType":"RunnerScaleSetJobMessages","body":"[]"}` + for _, test := range []struct { + name string + body string + want bool + }{ + {name: "all fields", body: full, want: true}, + {name: "empty object", body: empty, want: false}, + {name: "missing object", body: missing, want: false}, + } { + t.Run(test.name, func(t *testing.T) { + hook := newDrainPollHook("http://fixture.invalid/queue") + hook.inner = drainRoundTripper(func(*http.Request) (*http.Response, error) { + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(test.body))}, nil + }) + req, err := http.NewRequest(http.MethodGet, "http://fixture.invalid/queue", nil) + if err != nil { + t.Fatal(err) + } + response, err := hook.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + hook.releaseResponse() + if _, err := io.ReadAll(response.Body); err != nil { + t.Fatal(err) + } + if err := response.Body.Close(); err != nil { + t.Fatal(err) + } + _, got := hook.pollStatistics(1) + if got != test.want { + t.Fatalf("statistics field presence = %v, want %v", got, test.want) + } + }) + } } func TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine(t *testing.T) { @@ -531,6 +650,12 @@ type drainDriverAPI struct { sessionStats *scaleset.RunnerScaleSetStatistic } +type drainRoundTripper func(*http.Request) (*http.Response, error) + +func (f drainRoundTripper) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} + func (a *drainDriverAPI) GetScaleSet(context.Context, int) (*scaleset.RunnerScaleSet, error) { a.getScaleSetCalls++ stats := a.snapshotStats diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index d999153..0147f9a 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -192,15 +192,39 @@ func (d *Driver) record(e Event) error { return nil } +// cancellationFence prevents a phase effect after cancellation is visible at +// the durable-intent boundary. It cannot revoke bytes already accepted by a +// remote service; the final check immediately before the call only narrows +// the local race and keeps context-insensitive SDK fakes from running after a +// cancellation observed by this process. +func (d *Driver) cancellationFence(ctx context.Context, operation string) error { + if ctx != nil && ctx.Err() == nil { + return nil + } + if d.record(Event{Kind: "unknown", Operation: operation}) != nil { + return ErrJournal + } + return ErrQuarantine +} + // effect persists intent before an effect or work-bearing observation. A read // result can reveal work that must survive restart, so it has the same ordering. // A valid create/session identity and its work category share one result record. func (d *Driver) effect(ctx context.Context, op string, ids []int64, call func(context.Context) (Event, error)) error { + if err := d.cancellationFence(ctx, op); err != nil { + return err + } if err := d.record(Event{Kind: "intent", Operation: op, RequestIDs: ids}); err != nil { return err } + if err := d.cancellationFence(ctx, op); err != nil { + return err + } bounded, cancel := context.WithTimeout(ctx, operationTimeout) defer cancel() + if err := d.cancellationFence(bounded, op); err != nil { + return err + } e, err := call(bounded) if err != nil { if d.record(Event{Kind: "unknown", Operation: op}) != nil { diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 6f19691..e7673e9 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -215,7 +215,7 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "messageId": 17, "messageType": "RunnerScaleSetJobMessages", "body": string(jobs), - "statistics": map[string]int{"totalAvailableJobs": 1, "totalAssignedJobs": 1, "totalRegisteredRunners": 1, "totalIdleRunners": 1}, + "statistics": map[string]int{"totalAvailableJobs": 1, "totalAcquiredJobs": 0, "totalAssignedJobs": 1, "totalRunningJobs": 0, "totalRegisteredRunners": 1, "totalBusyRunners": 0, "totalIdleRunners": 1}, }) return } @@ -223,7 +223,11 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { t.Errorf("next poll capacity = %q, want 0", r.Header.Get(scaleset.HeaderScaleSetMaxCapacity)) } polls++ + w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]any{ + "statistics": map[string]int{"totalAvailableJobs": 0, "totalAcquiredJobs": 0, "totalAssignedJobs": 0, "totalRunningJobs": 0, "totalRegisteredRunners": 1, "totalBusyRunners": 0, "totalIdleRunners": 1}, + }) case r.URL.Path == "/queue/17" && r.Method == http.MethodDelete: acks++ w.WriteHeader(http.StatusNoContent) From 6e1b2144cb92a1a53db3926dd0e924c646643dfa Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 19:34:32 +0900 Subject: [PATCH 10/63] fix(g01): reject ambiguous drain evidence --- experiments/g01-scaleset/livecanary/drain.go | 74 ++++++++++------ .../g01-scaleset/livecanary/drain_test.go | 88 +++++++++++++++++++ 2 files changed, 135 insertions(+), 27 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index e306f7c..b76d78e 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -101,6 +101,9 @@ var drainStatisticsFields = [...]string{ // object into a non-nil all-zero struct, so a pointer alone is not evidence // that the service supplied a complete statistics sample. func drainStatisticsFromBody(body []byte) (drainStatistics, bool) { + if !uniqueKeys(json.NewDecoder(bytes.NewReader(body))) { + return drainStatistics{}, false + } var envelope map[string]json.RawMessage if json.Unmarshal(body, &envelope) != nil { return drainStatistics{}, false @@ -109,6 +112,9 @@ func drainStatisticsFromBody(body []byte) (drainStatistics, bool) { if !ok || bytes.Equal(bytes.TrimSpace(raw), []byte("null")) { return drainStatistics{}, false } + if !uniqueKeys(json.NewDecoder(bytes.NewReader(raw))) { + return drainStatistics{}, false + } var fields map[string]json.RawMessage if json.Unmarshal(raw, &fields) != nil { return drainStatistics{}, false @@ -208,6 +214,10 @@ func sameDrainRunner(before, after *drainRunnerIdentity) bool { return before != nil && after != nil && *before == *after } +func sameDrainRunnerPartition(left, right drainStatistics) bool { + return left.Registered == right.Registered && left.Busy == right.Busy && left.Idle == right.Idle +} + func validDrainObservation(o *drainObservation) bool { if o == nil || o.Version != drainObservationVersion || (o.Outcome != drainOutcomeObserved && o.Outcome != drainOutcomeInconclusive) || o.InitialCapacity != drainInitialCapacity || o.WithdrawnCapacity != drainWithdrawnCapacity || o.ServerReceipt != drainServerReceiptUnproven || o.Sequence <= 0 || o.ObservedAt.IsZero() || len(o.Ordering) > 8 { return false @@ -221,6 +231,9 @@ func validDrainObservation(o *drainObservation) bool { if o.Outcome == drainOutcomeObserved && !sameDrainRunner(o.Before.Runner, o.After.Runner) { return false } + if o.Outcome == drainOutcomeObserved && !sameDrainRunnerPartition(o.Poll.Statistics, o.Before.Statistics) { + return false + } seen := map[string]bool{} for _, item := range o.Ordering { if item != "poll-old" && item != "ack" && item != "acquire" && item != "poll-zero" || seen[item] { @@ -271,6 +284,7 @@ type drainPollHook struct { pollAttempts int statistics [3]drainStatistics statisticsKnown [3]bool + wroteCallbacks [3]int onRequestWritten func() wroteOnce sync.Once responseOnce sync.Once @@ -343,23 +357,27 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { } trace := &httptrace.ClientTrace{WroteRequest: func(info httptrace.WroteRequestInfo) { + notify := false h.mu.Lock() + if attempt < 1 || attempt >= len(h.wroteCallbacks) { + h.invalid = true + } else { + h.wroteCallbacks[attempt]++ + notify = h.wroteCallbacks[attempt] == 1 + if h.wroteCallbacks[attempt] != 1 { + h.invalid = true + } + } if info.Err != nil { h.invalid = true } else { h.wroteRequest = true } h.mu.Unlock() - if info.Err == nil { - h.wroteOnce.Do(func() { - if h.onRequestWritten != nil { - h.onRequestWritten() - } - close(h.wrote) - }) - } else { - h.wroteOnce.Do(func() { close(h.wrote) }) + if notify && info.Err == nil && h.onRequestWritten != nil { + h.onRequestWritten() } + h.wroteOnce.Do(func() { close(h.wrote) }) }} response, err := h.inner.RoundTrip(req.WithContext(httptrace.WithClientTrace(req.Context(), trace))) if err != nil { @@ -516,7 +534,7 @@ func (h *drainPollHook) boundary() (string, bool, bool) { defer h.mu.Unlock() switch { case h.wroteRequest && h.responseHeld && !h.responseBeforeWrite: - return drainBoundaryRequestWritten, true, !h.invalid + return drainBoundaryRequestWritten, true, !h.invalid && h.wroteCallbacks[1] == 1 case h.responseBeforeWrite: return drainBoundaryResponseBeforeWrite, h.responseHeld, false default: @@ -546,16 +564,18 @@ func (b *drainHeldBody) Close() error { // drainClient records only high-level listener effects and the two bounded // polls. It never changes ACK or acquisition order. type drainClient struct { - inner listener.Client - validate func(context.Context, *scaleset.RunnerScaleSetMessage) error - obs *drainObservation - hook *drainPollHook - phaseCtx context.Context - reject func(string) error - mu sync.Mutex - polls int - messageID int - requestID int64 + inner listener.Client + validate func(context.Context, *scaleset.RunnerScaleSetMessage) error + obs *drainObservation + hook *drainPollHook + phaseCtx context.Context + reject func(string) error + ownedRunnerStats drainStatistics + ownedRunnerStatsKnown bool + mu sync.Mutex + polls int + messageID int + requestID int64 } type drainContextBinder interface { @@ -644,7 +664,7 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal return nil, ErrQuarantine } messageStats, statsErr := newDrainStatistics(message.Statistics) - if statsErr != nil || (c.hook != nil && stats != messageStats) || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { + if statsErr != nil || (c.hook != nil && stats != messageStats) || (index == 1 && c.ownedRunnerStatsKnown && !sameDrainRunnerPartition(stats, c.ownedRunnerStats)) || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { c.mu.Lock() c.obs.poll(index).Message = drainMessageUnknown c.mu.Unlock() @@ -768,13 +788,13 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho if initial.SessionID == [16]byte{} || initial.Statistics == nil { return obs, ErrQuarantine } - if stats, err := newDrainStatistics(initial.Statistics); err != nil { + initialStats, err := newDrainStatistics(initial.Statistics) + if err != nil { return obs, err - } else { - obs.Poll.Statistics = stats - obs.Poll.StatsKnown = true } - c := &drainClient{inner: client, obs: &obs, hook: hook} + obs.Poll.Statistics = initialStats + obs.Poll.StatsKnown = true + c := &drainClient{inner: client, obs: &obs, hook: hook, ownedRunnerStats: initialStats, ownedRunnerStatsKnown: true} if rejecter, ok := client.(interface{ reject(string) error }); ok { c.reject = rejecter.reject } @@ -862,7 +882,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho if obs.Boundary == "" { obs.Boundary, obs.ResponseHeld, _ = hook.boundary() } - if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld && obs.Poll.Message == drainMessagePresent && obs.Poll.StatsKnown && obs.Poll.Statistics != (drainStatistics{}) && obs.Poll.ACK == drainResponseSucceeded && obs.Poll.Acquisition == drainResponseSucceeded && obs.NextPoll.Message == drainMessageAbsent && obs.NextPoll.StatsKnown { + if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld && obs.Poll.Message == drainMessagePresent && obs.Poll.StatsKnown && obs.Poll.Statistics != (drainStatistics{}) && c.ownedRunnerStatsKnown && sameDrainRunnerPartition(obs.Poll.Statistics, c.ownedRunnerStats) && obs.Poll.ACK == drainResponseSucceeded && obs.Poll.Acquisition == drainResponseSucceeded && obs.NextPoll.Message == drainMessageAbsent && obs.NextPoll.StatsKnown { obs.Outcome = drainOutcomeObserved return obs, nil } diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 0b59090..adfbdd7 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -7,6 +7,7 @@ import ( "io" "net/http" "net/http/httptest" + "net/http/httptrace" "strconv" "strings" "sync" @@ -577,6 +578,93 @@ func TestDrainPollHookPreservesStatisticsFieldPresence(t *testing.T) { } } +func TestDrainStatisticsRejectsDuplicateJSONFields(t *testing.T) { + full := `{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}` + for _, test := range []struct { + name string + body string + }{ + {name: "duplicate-required", body: `{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1,"totalIdleRunners":1}}`}, + {name: "duplicate-casefolded-required", body: `{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1,"TotalIdleRunners":1}}`}, + {name: "duplicate-top-level", body: full[:len(full)-1] + `,"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`}, + } { + t.Run(test.name, func(t *testing.T) { + stats, known := drainStatisticsFromBody([]byte(test.body)) + if known || stats != (drainStatistics{}) { + t.Fatalf("ambiguous statistics accepted: stats=%+v known=%v body=%s", stats, known, test.body) + } + }) + } +} + +type duplicatePollWriteTransport struct{} + +func (duplicatePollWriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.WroteRequest == nil { + return nil, errors.New("missing client trace") + } + trace.WroteRequest(httptrace.WroteRequestInfo{}) + trace.WroteRequest(httptrace.WroteRequestInfo{}) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`)), + }, nil +} + +func TestDrainPollHookRejectsDuplicatePhysicalWrites(t *testing.T) { + target := "http://fixture.invalid/queue" + hook := newDrainPollHook(target) + hook.inner = duplicatePollWriteTransport{} + req, err := http.NewRequest(http.MethodGet, target, nil) + if err != nil { + t.Fatal(err) + } + response, err := hook.RoundTrip(req) + if err != nil { + t.Fatal(err) + } + hook.releaseResponse() + _, _ = io.ReadAll(response.Body) + _ = response.Body.Close() + _, _, proven := hook.boundary() + if proven { + t.Fatal("duplicate physical poll write was promoted to a proven boundary") + } +} + +func TestDrainObservationRejectsPollCountersContradictingOwnedRunner(t *testing.T) { + observation := validDrainTestObservation() + // A valid internal partition with zero registered runners still contradicts + // the exact one-runner idle prerequisite captured before the poll. + observation.Poll.Statistics = drainStatistics{Available: 1, Assigned: 1} + if validDrainObservation(&observation) { + t.Fatal("poll counters contradicting the one owned runner were promoted to observed") + } +} + +func TestDrainClientRejectsPollRunnerPartitionMismatchBeforeEffects(t *testing.T) { + inner := &drainGuardSession{ + initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), Statistics: &scaleset.RunnerScaleSetStatistic{}}, + message: &scaleset.RunnerScaleSetMessage{ + MessageID: 7, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalAvailableJobs: 1, TotalAssignedJobs: 1}, + JobAvailableMessages: []*scaleset.JobAvailable{{JobMessageBase: scaleset.JobMessageBase{RunnerRequestID: 11}}}, + }, + } + obs := drainObservation{Poll: drainPollObservation{Message: drainMessageUnknown}, NextPoll: drainPollObservation{Message: drainMessageUnknown}} + client := &drainClient{ + inner: inner, obs: &obs, phaseCtx: context.Background(), + ownedRunnerStats: drainStatistics{Registered: 1, Idle: 1}, ownedRunnerStatsKnown: true, + } + if _, err := client.GetMessage(context.Background(), 0, drainInitialCapacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("runner partition mismatch = %v, want quarantine", err) + } + if inner.ackCalls != 0 || inner.acquireCalls != 0 { + t.Fatalf("runner partition mismatch triggered effects: ack=%d acquire=%d", inner.ackCalls, inner.acquireCalls) + } +} + func TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine(t *testing.T) { a := approval() a.Phases = append(a.Phases, "drain") From 5d715c486c959ae67ca615c4eaea3e7e89ede556 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 9 Sep 2026 23:42:30 +0900 Subject: [PATCH 11/63] fix(g01): bind drain evidence and reject physical poll retries --- docs/evidence/g01-idle-drain.md | 245 ++++++++++++- experiments/g01-scaleset/livecanary/drain.go | 83 +++-- .../g01-scaleset/livecanary/drain_driver.go | 8 + .../livecanary/drain_followup_test.go | 344 ++++++++++++++++++ .../g01-scaleset/livecanary/drain_test.go | 210 ++++++++++- experiments/g01-scaleset/livecanary/driver.go | 78 +++- .../g01-scaleset/livecanary/journal.go | 2 +- .../livecanary/sdk_integration_test.go | 12 + .../livecanary/security_review_extra_test.go | 169 +++++++++ 9 files changed, 1096 insertions(+), 55 deletions(-) create mode 100644 experiments/g01-scaleset/livecanary/drain_followup_test.go create mode 100644 experiments/g01-scaleset/livecanary/security_review_extra_test.go diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index d53d336..5ddbdc3 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -14,13 +14,13 @@ contradictory counters, a duplicate/wrong callback, or a second poll that returns a message after withdrawal. Each is recorded as inconclusive/quarantined and never treated as proof of a drain barrier. -The transport hook records `WroteRequest` and holds the first response body -before the high-level listener parser receives it. `WroteRequest` is a -client-side transport fact only: `server_receipt` remains `unproven`, so the -synthetic test and any future live result cannot claim server acceptance or an -atomic server-side drain. The implementation calls the released listener's -public `SetMaxRunners(0)` callback and lets the listener perform its normal -ACK-before-acquire sequence. +The transport hook records `WroteRequest` for each physical attempt in both +bounded polls and holds the first response body before the high-level listener +parser receives it. `WroteRequest` is a client-side transport fact only: +`server_receipt` remains `unproven`, so the synthetic test and any future live +result cannot claim server acceptance or an atomic server-side drain. The +implementation calls the released listener's public `SetMaxRunners(0)` +callback and lets the listener perform its normal ACK-before-acquire sequence. ## Bounded observation @@ -84,18 +84,97 @@ bounded adapter field-presence test; the cancellation fence is deliberately not described as atomic against a remote call that was already issued or accepted. +### Exact-head follow-up corrections + +This bounded follow-up started from exact base +`6e1b2144cb92a1a53db3926dd0e924c646643dfa` for PR #72. The current-head +Codex inventory was read with the local review script using these exact +invocations (the personal script path is intentionally omitted from committed +evidence): + +```text +bash /path/to/codex-review.sh all 72 --repo 1XP-AI/gh-runnerd +bash /path/to/codex-review.sh detail-all 72 --repo 1XP-AI/gh-runnerd +``` + +The actionable findings were [P1 withdrawn-poll physical retries](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496285), +[P2 drain-phase fence discharge](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496296), +and [P1 missing durable resolution evidence](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496305). +The first two were reproduced before the correction with this real failing +regression run: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestReplayDischargesCompletedDrainPhaseFence|TestDrainListenerRejectsWithdrawnPollPhysicalRetry' -count=1 -v +``` + +It failed with the valid final observation still reporting `uncertain: true` +and the synthetic listener promoting a transparent retry on the withdrawn +poll to `Outcome: observed` (`WroteRequest` callbacks `[0 1 0]`, `err=`). +The minimal correction stores the drain phase fence separately, clears only +that fence after a valid matching observed record, and traces both polls; +one successful physical write per poll is required, while error, duplicate and +transparent-retry callbacks remain invalid. The first poll's SDK response and +ACK-before-acquire path are unchanged. + +The focused green chronology was: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestReplayDischargesCompletedDrainPhaseFence|TestReplayDrainFence|TestDrainPollHookRequiresOneSuccessfulWritePerPoll|TestDrainListenerRejectsWithdrawnPollPhysicalRetry' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestDrain|TestReplayDischargesCompletedDrainPhaseFence|TestReplayDrainFence' -count=1 +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 +``` + +All three commands passed after the correction; the first includes the 4x4 +first/second poll matrix (success, write error, duplicate callback and +transparent retry), and the real listener fixture remained inconclusive for a +withdrawn-poll retry. These are offline tests only. + +The prior [P1 transparent first-poll retry finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770569) +and [P1 contradictory poll-counter finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770561) +remain part of the durable chronology. Their historical red preceded the +earlier correction; the current follow-up re-ran the regression guards with: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestDrainPollHookRejectsDuplicatePhysicalWrites|TestDrainPollHookRequiresOneSuccessfulWritePerPoll' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestDrainObservationRejectsPollCountersContradictingOwnedRunner|TestDrainClientRejectsPollRunnerPartitionMismatchBeforeEffects|TestDrainRequiresKnownConsistentStatisticsAndControlledMessage' -count=1 -v +``` + +Both commands passed on the corrected source. The first-poll guard counts +every physical callback and rejects retries; the counter guards reject a +valid-but-contradictory runner partition before ACK or acquisition. The +[duplicate-key regression](https://github.com/1XP-AI/gh-runnerd/commit/6e1b2144cb92a1a53db3926dd0e924c646643dfa) +is also retained and was re-run with: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestDrainStatisticsRejectsDuplicateJSONFields|TestDrainPollHookRejectsDuplicatePhysicalWrites' -count=1 -v +``` + +That command passed; duplicate or case-folded JSON keys remain unknown rather +than being accepted as a complete statistics sample. The historical reds are +not relabeled as pre-implementation tests for this follow-up; the actual red +run above is the new TDD regression, followed by the listed green runs. + ## Independent correction matrix | Finding | Correction and evidence | |---|---| -| Codex P1 status-only response close (`r3965776832`) | `drainHeldBody.Close` remains release-gated; `TestDrainHeldBodyHoldsCloseUntilRelease` passes (fix `f44f5f9`). | -| Codex P1 cancellation join (`r3965776826`) | all listener context exits release and join the run goroutine; `TestDrainRejectsEffectsAfterCancellationAndRecordsMarker` passes (fixes `f44f5f9`, `82d0d7d`). | -| Codex P1 drain route unreachable (`r3965776815`) | `Run` routes `drain` before generic owned/statistics quarantine; `TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine` passes (fix `f44f5f9`). | -| Codex P2 paired verification (`r3965957331`) | drain is admitted as a verification phase in paired approval/preparation; paired tests pass (fix `945371e`). | -| Codex P1 runner continuity (`r3966125441`) | observed requires exact before/after runner tuple; runner mutation is rejected (fix `6e954cf`). | -| Codex P1 embedded session set (`r3966362999`) | session-open now requires exact nested set identity, update fence, labels, and statistics equal to the idle before snapshot. `TestDrainRejectsEmbeddedSessionStatisticsMismatch` passes (fix `82d0d7d`). | -| Codex P1 ambiguous session retention (`r3966363009`) | non-successful listener outcomes retain the live session, write a quarantine marker, and never invoke session close; `TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine` asserts zero close calls (fix `82d0d7d`). | -| Codex P1 phase crash fence (`r3966362990`) | replay marks a durable `drain` phase uncertain before polling, so a crash before the final observation cannot authorize cleanup. `TestDrainPhaseStartRetainsCrashUncertainty` passes (fix `82d0d7d`). | +| Codex P1 [status-only response close](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3965776832) | `drainHeldBody.Close` remains release-gated; `TestDrainHeldBodyHoldsCloseUntilRelease` passes (fix `f44f5f9`). | +| Codex P1 [cancellation join](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3965776826) | all listener context exits release and join the run goroutine; `TestDrainRejectsEffectsAfterCancellationAndRecordsMarker` passes (fixes `f44f5f9`, `82d0d7d`). | +| Codex P1 [drain route unreachable](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3965776815) | `Run` routes `drain` before generic owned/statistics quarantine; `TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine` passes (fix `f44f5f9`). | +| Codex P2 [paired verification](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3965957331) | drain is admitted as a verification phase in paired approval/preparation; paired tests pass (fix `945371e`). | +| Codex P1 [runner continuity](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966125441) | observed requires exact before/after runner tuple; runner mutation is rejected (fix `6e954cf`). | +| Codex P1 [embedded session set](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966362999) | session-open now requires exact nested set identity, update fence, labels, and statistics equal to the idle before snapshot. `TestDrainRejectsEmbeddedSessionStatisticsMismatch` passes (fix `82d0d7d`). | +| Codex P1 [ambiguous session retention](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966363009) | non-successful listener outcomes retain the live session, write a quarantine marker, and never invoke session close; `TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine` asserts zero close calls (fix `82d0d7d`). | +| Codex P1 [phase crash fence](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966362990) | replay marks a durable `drain` phase uncertain before polling, so a crash before the final observation cannot authorize cleanup. `TestDrainPhaseStartRetainsCrashUncertainty` passes (fix `82d0d7d`). | +| Codex P1 [transparent first-poll retry](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770569) | every first-poll physical `WroteRequest` callback is counted; duplicates/errors remain invalid. `TestDrainPollHookRejectsDuplicatePhysicalWrites` and the 4x4 two-poll matrix pass. | +| Codex P1 [contradictory poll counters](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770561) | poll runner partitions must match the owned idle prerequisite before ACK/acquisition; `TestDrainObservationRejectsPollCountersContradictingOwnedRunner` and `TestDrainClientRejectsPollRunnerPartitionMismatchBeforeEffects` pass. | +| Codex P1 [withdrawn-poll retry](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496285) | both bounded polls are traced; any second-poll error/duplicate/retry prevents `observed`, and the actual listener fixture remains inconclusive. `TestDrainListenerRejectsWithdrawnPollPhysicalRetry` passes. | +| Codex P2 [drain-phase fence discharge](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496296) | replay discharges only a valid matching observed drain record's phase-local fence; unrelated uncertainty, work, reservations, and failed/inconclusive/crashed fences survive. `TestReplayDischargesCompletedDrainPhaseFence`, `TestReplayDrainFencePreservesUnrelatedUncertaintyAndReservations`, and `TestReplayDrainFenceRetainsInconclusiveOutcome` pass. | +| Codex P1 [durable finding evidence](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496305) | this section and the matrix record full finding URLs, exact red/green commands, actual outcomes and rollback scope for the current and prior corrections. | | Security F1 / Protocol F1 poll reservation | `observe-poll` result journals one verified request ID plus fixed work before ACK; replay sets reservation/work fences. `TestDrainPollJournalsReservationBeforeACK` passes (`82d0d7d`). | | Security F2 rejected idle prerequisite | complete bounded `DrainSnapshot` plus `prerequisite-failed` marker is retained; replay remains uncertain. `TestDrainRejectedIdlePrerequisiteRetainsFence` passes (`82d0d7d`). | | Security F3 / Protocol F2 runner and nested set identity | observed requires exact runner continuity; session-open validates nested set ID/name/group/label/update fence. Pinned SDK drain integration and mutation tests pass (`6e954cf`, `82d0d7d`). | @@ -136,12 +215,142 @@ qualification. No credentials, runner/session/JIT operation, workflow operation, app/keychain/launchd/Docker/Lima mutation or cleanup was performed. +### Exact-head blocker corrections (local uncommitted tree) + +Before edits, the independent review probes were re-run against the immutable +`1eb48afb50ffbb10b42d07181f16153df1c494eb` source extraction with: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestIndependentReplayPhaseBindingBoundaries|TestIndependentFileJournalStoresMismatchedDrainSequence|TestIndependentContradictoryNextPollStatsCannotPromoteDrain' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestSecurityReviewReplayIdentitySequencePhaseBoundaries|TestSecurityReviewValidObservationNeedsExactRequiredFields|TestSecurityReviewDrainObservationDoesNotAuthorizeLaterWorkOrCleanup' -count=1 -v +``` + +Both commands failed as expected. Replay accepted absent, repeated, +interrupted, foreign-set and `Sequence=999` histories; the durable FileJournal +reopen accepted `Sequence=999`; and the listener promoted a known withdrawn +poll partition of `registered=0,busy=0,idle=0` to `observed`. The probes also +confirmed the existing runner continuity, byte/body budget, retry trace and +ACK-before-acquisition controls remained intact before this correction. + +The local red-first correction adds phase-local replay binding and the +withdrawn-poll runner-partition fence. A drain phase records its created +scale-set ID, and its journal-assigned event sequence becomes the only valid +`Drain.Sequence`; replay requires exactly one active phase, the created set ID +in both snapshots, and a later matching observation. Both polls and the final +snapshot compare only `registered`, `busy` and `idle` runner counters, so job +counters remain free to change. + +The focused correction and positive controls were run from the experiment +module: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDrainObservedAllowsNextPollJobCounterChanges|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition|TestReplayDischargesCompletedDrainPhaseFence|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestDrain|TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +cd ../.. +bash scripts/gofmt.sh check +bash scripts/check-offline-experiments.sh +``` + +All commands passed. The first includes the real FileJournal close/reopen +checks and the real pinned `github.com/actions/scaleset v0.4.0` listener path; +the full package, focused race, vet, formatting and offline experiment checks +also passed. These remain offline tests only, and the local correction is +intentionally uncommitted and unpublished for coordinator exact-head review. + +### Missing drain-phase SetID correction (current local uncommitted tree) + +The red-first regression was added before the implementation change and run +with: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestReplayDrainPhaseRequiresPositiveSetID|TestFileJournalRejectsNonPositiveDrainPhaseSetID|TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen' -count=1 -v +``` + +It failed as expected: replay discharged matching observations for missing and +zero phase IDs, and the real FileJournal accepted missing, zero and negative +drain-phase IDs. Negative and foreign-positive direct replay histories already +remained fenced. + +The minimal correction makes `validEvent` require `ID > 0` for +`phase/drain`, prevents replay from inferring a missing, zero or negative ID +from the prior create result, and retains uncertainty for foreign-positive +phase IDs. The Driver's positive phase ID is now asserted through the pinned +SDK journal path together with exact phase-sequence observation binding. + +The focused green and verification commands were: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestReplayDrainPhaseRequiresPositiveSetID|TestFileJournalRejectsNonPositiveDrainPhaseSetID|TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen|TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDrainObservedAllowsNextPollJobCounterChanges|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition|TestReplayDischargesCompletedDrainPhaseFence|TestReplayDrainFencePreservesUnrelatedUncertaintyAndReservations|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestReplayDrainPhaseRequiresPositiveSetID|TestFileJournalRejectsNonPositiveDrainPhaseSetID|TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen|TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition|TestReplayDischargesCompletedDrainPhaseFence|TestReplayDrainFencePreservesUnrelatedUncertaintyAndReservations|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +gofmt -l experiments/g01-scaleset/livecanary/journal.go experiments/g01-scaleset/livecanary/driver.go experiments/g01-scaleset/livecanary/drain_followup_test.go experiments/g01-scaleset/livecanary/sdk_integration_test.go +git diff --check +``` + +All listed verification commands passed; the formatting and diff checks emitted +no output. No broad multi-module gate was rerun for this narrow correction. +These are offline tests only; no live runner, workflow, credential, journal +cleanup, or external operation was performed. + +### Security F2 owned-idle prerequisite correction (current local uncommitted tree) + +The prior F1 phase-ID correction remains in place; its separate red/green +evidence above is unchanged and was included in the focused verification below. +The new F2 red-first regression was run before its implementation with: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestSecurityReviewDrainRequiresOwnedIdleBeforeProof|TestSecurityReviewMatchingDrainPhaseDischargesAfterFileJournalReopen|TestSecurityReviewValidOneIdleRunnerDrainObservation|TestSecurityReviewInconclusiveDrainMayHaveMissingOwnedIdentity|TestSecurityReviewObservedDrainAllowsLegitimateJobCounterChanges' -count=1 -v +``` + +It failed as expected: observed two-runner and busy snapshots were accepted, +and the real FileJournal accepted a positive-ID, correctly sequenced +non-owned-idle observation and discharged the replay fence. Missing owned +identity already remained rejected, while the valid one-idle, inconclusive, +and legitimate job-counter controls passed. + +The minimal F2 correction adds the existing `validDrainIdlePrerequisite` to +the `drainOutcomeObserved` branch of `validDrainObservation`; the runner +partition equality checks remain unchanged. Inconclusive observations may +still carry missing runner identity, while observed evidence now requires the +owned one-runner idle proof before journal append or replay can discharge a +phase fence. + +The focused green, race, pinned-SDK, vet, formatting, and diff checks were: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestSecurityReviewDrainRequiresOwnedIdleBeforeProof|TestSecurityReviewMatchingDrainPhaseDischargesAfterFileJournalReopen|TestSecurityReviewValidOneIdleRunnerDrainObservation|TestSecurityReviewInconclusiveDrainMayHaveMissingOwnedIdentity|TestSecurityReviewObservedDrainAllowsLegitimateJobCounterChanges|TestReplayDrainPhaseRequiresPositiveSetID|TestFileJournalRejectsNonPositiveDrainPhaseSetID|TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen|TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDrainObservedAllowsNextPollJobCounterChanges|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition|TestReplayDischargesCompletedDrainPhaseFence|TestReplayDrainFencePreservesUnrelatedUncertaintyAndReservations|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestSecurityReviewDrainRequiresOwnedIdleBeforeProof|TestSecurityReviewMatchingDrainPhaseDischargesAfterFileJournalReopen|TestSecurityReviewValidOneIdleRunnerDrainObservation|TestSecurityReviewInconclusiveDrainMayHaveMissingOwnedIdentity|TestSecurityReviewObservedDrainAllowsLegitimateJobCounterChanges|TestReplayDrainPhaseRequiresPositiveSetID|TestFileJournalRejectsNonPositiveDrainPhaseSetID|TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen|TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition|TestReplayDischargesCompletedDrainPhaseFence|TestReplayDrainFencePreservesUnrelatedUncertaintyAndReservations|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +gofmt -l experiments/g01-scaleset/livecanary/drain.go experiments/g01-scaleset/livecanary/journal.go experiments/g01-scaleset/livecanary/driver.go experiments/g01-scaleset/livecanary/drain_followup_test.go experiments/g01-scaleset/livecanary/security_review_extra_test.go experiments/g01-scaleset/livecanary/sdk_integration_test.go +git diff --check +``` + +All listed checks passed; the race run reported no race, and formatting and +diff checks emitted no output. The pinned `github.com/actions/scaleset v0.4.0` +Driver path remains offline-only, no broad multi-module gate was repeated, and +no live operation or personal path was added. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an immutable reviewed head, the approved private repository/workflow/resources, one idle owned canary runner and independent review. A timing miss, missing runner, stale zero, identity/statistics mismatch, unknown response, or -server-receipt ambiguity must remain inconclusive. Rollback is the focused -revert of the issue-71 commit/PR; no live cleanup or workflow replay is part -of rollback. +server-receipt ambiguity must remain inconclusive. Rollback of the latest +follow-up is a focused revert of its code/test/documentation commit(s), with +the exact current journal and owned resources retained for inspection. If an +earlier correction must be isolated, revert only the reviewed source slice for +[r3966770569](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770569), +[r3966770561](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770561), +or the [duplicate-key regression](https://github.com/1XP-AI/gh-runnerd/commit/6e1b2144cb92a1a53db3926dd0e924c646643dfa) +after checking dependent corrections; do not reset, erase or replay the +journal. No live cleanup, workflow replay, runner mutation or rollback +operation was performed. diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index b76d78e..3519b67 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -231,9 +231,15 @@ func validDrainObservation(o *drainObservation) bool { if o.Outcome == drainOutcomeObserved && !sameDrainRunner(o.Before.Runner, o.After.Runner) { return false } + if o.Outcome == drainOutcomeObserved && !validDrainIdlePrerequisite(o.Before) { + return false + } if o.Outcome == drainOutcomeObserved && !sameDrainRunnerPartition(o.Poll.Statistics, o.Before.Statistics) { return false } + if o.Outcome == drainOutcomeObserved && (!sameDrainRunnerPartition(o.NextPoll.Statistics, o.Before.Statistics) || !sameDrainRunnerPartition(o.After.Statistics, o.Before.Statistics)) { + return false + } seen := map[string]bool{} for _, item := range o.Ordering { if item != "poll-old" && item != "ack" && item != "acquire" && item != "poll-zero" || seen[item] { @@ -333,11 +339,14 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { if !second { return nil, ErrQuarantine } - response, err := h.inner.RoundTrip(req) + response, err := h.inner.RoundTrip(h.tracedRequest(req, attempt)) if err != nil { return nil, err } if response == nil { + h.mu.Lock() + h.invalid = true + h.mu.Unlock() return nil, ErrQuarantine } if response.Body == nil { @@ -356,30 +365,7 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { return response, nil } - trace := &httptrace.ClientTrace{WroteRequest: func(info httptrace.WroteRequestInfo) { - notify := false - h.mu.Lock() - if attempt < 1 || attempt >= len(h.wroteCallbacks) { - h.invalid = true - } else { - h.wroteCallbacks[attempt]++ - notify = h.wroteCallbacks[attempt] == 1 - if h.wroteCallbacks[attempt] != 1 { - h.invalid = true - } - } - if info.Err != nil { - h.invalid = true - } else { - h.wroteRequest = true - } - h.mu.Unlock() - if notify && info.Err == nil && h.onRequestWritten != nil { - h.onRequestWritten() - } - h.wroteOnce.Do(func() { close(h.wrote) }) - }} - response, err := h.inner.RoundTrip(req.WithContext(httptrace.WithClientTrace(req.Context(), trace))) + response, err := h.inner.RoundTrip(h.tracedRequest(req, attempt)) if err != nil { h.wroteOnce.Do(func() { close(h.wrote) }) h.responseOnce.Do(func() { close(h.response) }) @@ -419,6 +405,42 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { return response, nil } +func (h *drainPollHook) tracedRequest(req *http.Request, attempt int) *http.Request { + trace := &httptrace.ClientTrace{WroteRequest: func(info httptrace.WroteRequestInfo) { + notify := false + h.mu.Lock() + if attempt < 1 || attempt >= len(h.wroteCallbacks) { + h.invalid = true + } else { + h.wroteCallbacks[attempt]++ + notify = attempt == 1 && h.wroteCallbacks[attempt] == 1 + if h.wroteCallbacks[attempt] != 1 { + h.invalid = true + } + } + if info.Err != nil { + h.invalid = true + } else if attempt == 1 { + h.wroteRequest = true + } + h.mu.Unlock() + if notify && info.Err == nil && h.onRequestWritten != nil { + h.onRequestWritten() + } + h.wroteOnce.Do(func() { close(h.wrote) }) + }} + return req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) +} + +func (h *drainPollHook) pollWritesValid() bool { + if h == nil { + return false + } + h.mu.Lock() + defer h.mu.Unlock() + return !h.invalid && h.wroteCallbacks[1] == 1 && h.wroteCallbacks[2] == 1 +} + func (h *drainPollHook) pollStatistics(index int) (drainStatistics, bool) { if h == nil || index < 1 || index >= len(h.statistics) { return drainStatistics{}, false @@ -649,6 +671,13 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal } if message == nil { c.mu.Lock() + if statsKnown && c.ownedRunnerStatsKnown && !sameDrainRunnerPartition(stats, c.ownedRunnerStats) { + c.obs.poll(index).Message = drainMessageUnknown + c.obs.poll(index).Statistics = drainStatistics{} + c.obs.poll(index).StatsKnown = false + c.mu.Unlock() + return nil, ErrQuarantine + } c.obs.poll(index).Message = drainMessageAbsent c.obs.poll(index).Statistics = stats c.obs.poll(index).StatsKnown = statsKnown @@ -664,7 +693,7 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal return nil, ErrQuarantine } messageStats, statsErr := newDrainStatistics(message.Statistics) - if statsErr != nil || (c.hook != nil && stats != messageStats) || (index == 1 && c.ownedRunnerStatsKnown && !sameDrainRunnerPartition(stats, c.ownedRunnerStats)) || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { + if statsErr != nil || (c.hook != nil && stats != messageStats) || (index <= 2 && c.ownedRunnerStatsKnown && !sameDrainRunnerPartition(stats, c.ownedRunnerStats)) || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { c.mu.Lock() c.obs.poll(index).Message = drainMessageUnknown c.mu.Unlock() @@ -882,7 +911,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho if obs.Boundary == "" { obs.Boundary, obs.ResponseHeld, _ = hook.boundary() } - if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld && obs.Poll.Message == drainMessagePresent && obs.Poll.StatsKnown && obs.Poll.Statistics != (drainStatistics{}) && c.ownedRunnerStatsKnown && sameDrainRunnerPartition(obs.Poll.Statistics, c.ownedRunnerStats) && obs.Poll.ACK == drainResponseSucceeded && obs.Poll.Acquisition == drainResponseSucceeded && obs.NextPoll.Message == drainMessageAbsent && obs.NextPoll.StatsKnown { + if obs.Boundary == drainBoundaryRequestWritten && obs.ResponseHeld && hook.pollWritesValid() && obs.Poll.Message == drainMessagePresent && obs.Poll.StatsKnown && obs.Poll.Statistics != (drainStatistics{}) && c.ownedRunnerStatsKnown && sameDrainRunnerPartition(obs.Poll.Statistics, c.ownedRunnerStats) && obs.Poll.ACK == drainResponseSucceeded && obs.Poll.Acquisition == drainResponseSucceeded && obs.NextPoll.Message == drainMessageAbsent && obs.NextPoll.StatsKnown && sameDrainRunnerPartition(obs.NextPoll.Statistics, c.ownedRunnerStats) { obs.Outcome = drainOutcomeObserved return obs, nil } diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 96b72d3..a8b1ccd 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -196,6 +196,11 @@ func (d *Driver) drain(ctx context.Context, setID int) error { if !ok { return ErrApproval } + phaseState := replay(d.Journal.Events()) + if !phaseState.drainPhasePending || phaseState.drainPhaseSequence <= 0 || phaseState.drainPhaseSetID != setID { + return ErrQuarantine + } + phaseSequence := phaseState.drainPhaseSequence before, err := d.drainSnapshot(ctx, setID, "before") if err != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, err)); markerErr != nil { @@ -238,6 +243,9 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID} obs, runErr := runDrainListener(ctx, journaled, setID, hook) + // The bounded observation sequence is phase-local: it must identify the + // exact durable drain phase that was active when the listener ran. + obs.Sequence = phaseSequence if ctx.Err() != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, runErr)); markerErr != nil { return markerErr diff --git a/experiments/g01-scaleset/livecanary/drain_followup_test.go b/experiments/g01-scaleset/livecanary/drain_followup_test.go new file mode 100644 index 0000000..3bee4d6 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/drain_followup_test.go @@ -0,0 +1,344 @@ +package livecanary + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + "net/http/httptrace" + "os" + "strings" + "testing" + + "github.com/actions/scaleset" + "github.com/google/uuid" +) + +func drainReplayPrefix() []Event { + return []Event{ + {Kind: "phase", Operation: "create", Sequence: 1}, + {Kind: "intent", Operation: "create", Sequence: 2}, + {Kind: "result", Operation: "create", Sequence: 3, ID: 7}, + {Kind: "phase", Operation: "drain", Sequence: 4, ID: 7}, + } +} + +func TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence(t *testing.T) { + base := validDrainTestObservation() + base.Sequence = 4 + foreign := base + foreign.Before.Set.ID = 99 + foreign.After.Set.ID = 99 + foreign.Before.Runner = &drainRunnerIdentity{ID: 19, Name: "g01-test-worker-1", ScaleSetID: 99} + foreign.After.Runner = &drainRunnerIdentity{ID: 19, Name: "g01-test-worker-1", ScaleSetID: 99} + + cases := []struct { + name string + events func(drainObservation) []Event + uncertain bool + }{ + { + name: "one matching phase", + events: func(observation drainObservation) []Event { + events := drainReplayPrefix() + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}) + return events + }, + uncertain: false, + }, + { + name: "missing phase", + events: func(observation drainObservation) []Event { + events := drainReplayPrefix()[:3] + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 4, Drain: &observation}) + }, + uncertain: true, + }, + { + name: "repeated phase", + events: func(observation drainObservation) []Event { + events := drainReplayPrefix() + events = append(events, Event{Kind: "phase", Operation: "drain", Sequence: 5, ID: 7}) + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 6, Drain: &observation}) + }, + uncertain: true, + }, + { + name: "interrupted phase", + events: func(observation drainObservation) []Event { + events := drainReplayPrefix() + events = append(events, Event{Kind: "phase", Operation: "inspect", Sequence: 5}) + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 6, Drain: &observation}) + }, + uncertain: true, + }, + { + name: "mismatched sequence", + events: func(observation drainObservation) []Event { + observation.Sequence = 999 + events := drainReplayPrefix() + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}) + }, + uncertain: true, + }, + { + name: "mismatched phase identity", + events: func(observation drainObservation) []Event { + events := drainReplayPrefix() + events[3].ID = 99 + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}) + }, + uncertain: true, + }, + { + name: "mismatched created identity", + events: func(observation drainObservation) []Event { + events := drainReplayPrefix() + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}) + }, + uncertain: true, + }, + { + name: "duplicate observation", + events: func(observation drainObservation) []Event { + events := drainReplayPrefix() + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}) + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 6, Drain: &observation}) + }, + uncertain: true, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + observation := base + if tc.name == "mismatched created identity" { + observation = foreign + } + if got := replay(tc.events(observation)).uncertain; got != tc.uncertain { + t.Fatalf("uncertain=%v, want %v; state=%+v", got, tc.uncertain, replay(tc.events(observation))) + } + }) + } +} + +func TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*drainObservation) + }{ + {name: "sequence", mutate: func(observation *drainObservation) { observation.Sequence = 999 }}, + {name: "created identity", mutate: func(observation *drainObservation) { + observation.Before.Set.ID = 99 + observation.After.Set.ID = 99 + observation.Before.Runner = &drainRunnerIdentity{ID: 19, Name: "g01-test-worker-1", ScaleSetID: 99} + observation.After.Runner = &drainRunnerIdentity{ID: 19, Name: "g01-test-worker-1", ScaleSetID: 99} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + directory := t.TempDir() + if err := os.Chmod(directory, 0700); err != nil { + t.Fatal(err) + } + j, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix() { + if err := j.Append(event); err != nil { + j.Close() + t.Fatalf("append prefix: %v", err) + } + } + observation := validDrainTestObservation() + if tc.name == "sequence" { + observation.Sequence = 999 + } else { + phase := j.Events()[len(j.Events())-1] + observation.Sequence = phase.Sequence + } + tc.mutate(&observation) + if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { + j.Close() + t.Fatalf("append drain observation: %v", err) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if state := replay(reopened.Events()); !state.uncertain { + t.Fatalf("reopened mismatched drain history discharged fence: %+v", state) + } + }) + } +} + +func TestReplayDrainPhaseRequiresPositiveSetID(t *testing.T) { + base := validDrainTestObservation() + base.Sequence = 4 + for _, tc := range []struct { + name string + id int + }{ + {name: "missing phase ID", id: 0}, + {name: "zero phase ID", id: 0}, + {name: "negative phase ID", id: -1}, + {name: "foreign phase ID", id: 99}, + } { + t.Run(tc.name, func(t *testing.T) { + events := drainReplayPrefix() + events[3].ID = tc.id + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &base}) + if state := replay(events); !state.uncertain { + t.Fatalf("drain phase with %d SetID discharged its fence: %+v", tc.id, state) + } + }) + } +} + +func TestFileJournalRejectsNonPositiveDrainPhaseSetID(t *testing.T) { + for _, tc := range []struct { + name string + id int + }{ + {name: "missing phase ID", id: 0}, + {name: "zero phase ID", id: 0}, + {name: "negative phase ID", id: -1}, + } { + t.Run(tc.name, func(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + directory := privateDir(t) + j, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + j.Close() + t.Fatalf("append create prefix: %v", err) + } + } + phase := drainReplayPrefix()[3] + phase.ID = tc.id + if err := j.Append(phase); err == nil { + j.Close() + t.Fatalf("accepted drain phase with non-positive SetID %d", tc.id) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if events := reopened.Events(); len(events) != 3 || replay(events).setID != 7 { + t.Fatalf("reopened journal after rejected drain phase = %+v", events) + } + }) + } +} + +func TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + directory := privateDir(t) + j, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + j.Close() + t.Fatalf("append create prefix: %v", err) + } + } + phase := drainReplayPrefix()[3] + phase.ID = 99 + if err := j.Append(phase); err != nil { + j.Close() + t.Fatalf("append foreign drain phase: %v", err) + } + observation := validDrainTestObservation() + observation.Sequence = phase.Sequence + if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { + j.Close() + t.Fatalf("append drain observation: %v", err) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if state := replay(reopened.Events()); !state.uncertain { + t.Fatalf("reopened foreign drain phase discharged its fence: %+v", state) + } +} + +func TestDrainObservedAllowsNextPollJobCounterChanges(t *testing.T) { + observation := validDrainTestObservation() + observation.NextPoll.Statistics.Available = 4 + observation.NextPoll.Statistics.Acquired = 3 + observation.NextPoll.Statistics.Assigned = 2 + observation.NextPoll.Statistics.Running = 1 + if !validDrainObservation(&observation) { + t.Fatal("job counters in the withdrawn poll were incorrectly frozen") + } +} + +type contradictoryNextPollTransport struct{ polls int } + +func (t *contradictoryNextPollTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodGet { + status := http.StatusNoContent + body := io.ReadCloser(http.NoBody) + if req.Method == http.MethodPost { + status = http.StatusOK + body = io.NopCloser(strings.NewReader(`{"count":1,"value":[11]}`)) + } + return &http.Response{StatusCode: status, Status: http.StatusText(status), Header: make(http.Header), Request: req, Body: body}, nil + } + t.polls++ + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.WroteRequest == nil { + return nil, errors.New("missing client trace") + } + trace.WroteRequest(httptrace.WroteRequestInfo{}) + body := `{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}` + status := http.StatusOK + if t.polls == 1 { + body = `{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[]"}` + } else { + status = http.StatusAccepted + body = `{"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":0,"totalBusyRunners":0,"totalIdleRunners":0}}` + } + return &http.Response{StatusCode: status, Status: http.StatusText(status), Header: make(http.Header), Request: req, Body: io.NopCloser(bytes.NewBufferString(body))}, nil +} + +func TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition(t *testing.T) { + const target = "http://fixture.invalid/queue" + hook := newDrainPollHook(target) + hook.inner = &contradictoryNextPollTransport{} + session := &drainSyntheticSession{ + client: &http.Client{Transport: hook}, + initial: scaleset.RunnerScaleSetSession{ + SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: target, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, + order: new([]string), + } + observation, err := runDrainListener(context.Background(), session, 7, hook) + if err == nil || observation.Outcome == drainOutcomeObserved { + t.Fatalf("contradictory withdrawn-poll runner partition was observed: observation=%+v err=%v", observation, err) + } +} diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index adfbdd7..5cf61c3 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -225,8 +225,8 @@ func TestDrainObservationIsBoundedAndFailClosed(t *testing.T) { } state := replay([]Event{{Kind: "observation", Operation: "drain", Drain: &observation}}) - if !state.workObserved || state.uncertain { - t.Fatalf("observed drain replay = %+v, want work retained without uncertainty", state) + if !state.workObserved || !state.uncertain { + t.Fatalf("unbound observed drain replay = %+v, want retained work and uncertainty", state) } observation.Outcome = drainOutcomeInconclusive state = replay([]Event{{Kind: "observation", Operation: "drain", Drain: &observation}}) @@ -287,6 +287,54 @@ func TestDrainPhaseStartRetainsCrashUncertainty(t *testing.T) { } } +func TestReplayDischargesCompletedDrainPhaseFence(t *testing.T) { + observation := validDrainTestObservation() + observation.Sequence = 4 + state := replay([]Event{ + {Kind: "phase", Operation: "create", Sequence: 1}, + {Kind: "intent", Operation: "create", Sequence: 2}, + {Kind: "result", Operation: "create", Sequence: 3, ID: 7}, + {Kind: "phase", Operation: "drain", Sequence: 4, ID: 7}, + {Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}, + }) + if state.uncertain { + t.Fatalf("valid drain observation retained its completed phase fence: %+v", state) + } +} + +func TestReplayDrainFencePreservesUnrelatedUncertaintyAndReservations(t *testing.T) { + observation := validDrainTestObservation() + state := replay([]Event{ + {Kind: "phase", Operation: "drain"}, + {Kind: "intent", Operation: "acquire", RequestIDs: []int64{41}}, + {Kind: "observation", Operation: "drain", Drain: &observation}, + }) + if !state.uncertain || !state.reserved || !state.workObserved { + t.Fatalf("completed drain phase cleared unrelated fences: %+v", state) + } + + state = replay([]Event{ + {Kind: "phase", Operation: "drain"}, + {Kind: "observation", Operation: "drain", Drain: &observation}, + {Kind: "unknown", Operation: "acquire"}, + }) + if !state.uncertain { + t.Fatalf("completed drain phase cleared a later unknown fence: %+v", state) + } +} + +func TestReplayDrainFenceRetainsInconclusiveOutcome(t *testing.T) { + observation := validDrainTestObservation() + observation.Outcome = drainOutcomeInconclusive + state := replay([]Event{ + {Kind: "phase", Operation: "drain"}, + {Kind: "observation", Operation: "drain", Drain: &observation}, + }) + if !state.uncertain { + t.Fatalf("inconclusive drain phase discharged its fence: %+v", state) + } +} + func TestDrainPollJournalsReservationBeforeACK(t *testing.T) { a := approval() j := &memoryJournal{} @@ -612,6 +660,101 @@ func (duplicatePollWriteTransport) RoundTrip(req *http.Request) (*http.Response, }, nil } +type drainPhysicalWriteBehavior string + +const ( + drainWriteSuccess drainPhysicalWriteBehavior = "success" + drainWriteError drainPhysicalWriteBehavior = "error" + drainWriteDuplicate drainPhysicalWriteBehavior = "duplicate" + drainWriteRetry drainPhysicalWriteBehavior = "retry" +) + +type drainPhysicalWriteTransport struct { + first drainPhysicalWriteBehavior + second drainPhysicalWriteBehavior + polls int +} + +func (t *drainPhysicalWriteTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodGet { + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody}, nil + } + t.polls++ + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.WroteRequest == nil { + return nil, errors.New("missing client trace") + } + behavior := t.first + if t.polls == 2 { + behavior = t.second + } + switch behavior { + case drainWriteSuccess: + trace.WroteRequest(httptrace.WroteRequestInfo{}) + case drainWriteError: + trace.WroteRequest(httptrace.WroteRequestInfo{Err: errors.New("synthetic physical write error")}) + return nil, errors.New("synthetic transport error") + case drainWriteDuplicate: + trace.WroteRequest(httptrace.WroteRequestInfo{}) + trace.WroteRequest(httptrace.WroteRequestInfo{}) + case drainWriteRetry: + trace.WroteRequest(httptrace.WroteRequestInfo{Err: errors.New("synthetic transparent retry")}) + trace.WroteRequest(httptrace.WroteRequestInfo{}) + default: + return nil, errors.New("unknown synthetic write behavior") + } + if t.polls == 1 { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`)), + }, nil + } + return &http.Response{ + StatusCode: http.StatusAccepted, + Body: io.NopCloser(strings.NewReader(`{"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`)), + }, nil +} + +type secondPollRetryTransport struct { + polls int +} + +func (t *secondPollRetryTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodGet { + status := http.StatusOK + body := io.ReadCloser(http.NoBody) + if req.Method == http.MethodDelete { + status = http.StatusNoContent + } else { + body = io.NopCloser(strings.NewReader(`{"count":1,"value":[11]}`)) + } + return &http.Response{StatusCode: status, Status: http.StatusText(status), Body: body}, nil + } + t.polls++ + trace := httptrace.ContextClientTrace(req.Context()) + if t.polls == 1 { + if trace == nil || trace.WroteRequest == nil { + return nil, errors.New("missing first-poll client trace") + } + trace.WroteRequest(httptrace.WroteRequestInfo{}) + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[]"}`)), + }, nil + } + if t.polls != 2 { + return nil, errors.New("unexpected extra poll") + } + if trace != nil && trace.WroteRequest != nil { + trace.WroteRequest(httptrace.WroteRequestInfo{Err: errors.New("synthetic transparent retry")}) + trace.WroteRequest(httptrace.WroteRequestInfo{}) + } + return &http.Response{ + StatusCode: http.StatusAccepted, + Body: io.NopCloser(strings.NewReader(`{"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`)), + }, nil +} + func TestDrainPollHookRejectsDuplicatePhysicalWrites(t *testing.T) { target := "http://fixture.invalid/queue" hook := newDrainPollHook(target) @@ -633,6 +776,69 @@ func TestDrainPollHookRejectsDuplicatePhysicalWrites(t *testing.T) { } } +func TestDrainListenerRejectsWithdrawnPollPhysicalRetry(t *testing.T) { + const target = "http://fixture.invalid/queue" + hook := newDrainPollHook(target) + transport := &secondPollRetryTransport{} + hook.inner = transport + session := &drainSyntheticSession{ + client: &http.Client{Transport: hook}, + initial: scaleset.RunnerScaleSetSession{ + SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: target, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, + order: new([]string), + } + observation, err := runDrainListener(context.Background(), session, 7, hook) + if transport.polls != 2 { + t.Fatalf("polls = %d, want first and withdrawn polls", transport.polls) + } + if err == nil || observation.Outcome == drainOutcomeObserved { + t.Fatalf("withdrawn-poll physical retry was promoted to observed: observation=%+v err=%v", observation, err) + } +} + +func TestDrainPollHookRequiresOneSuccessfulWritePerPoll(t *testing.T) { + behaviors := []drainPhysicalWriteBehavior{drainWriteSuccess, drainWriteError, drainWriteDuplicate, drainWriteRetry} + for _, first := range behaviors { + for _, second := range behaviors { + t.Run(string(first)+"/"+string(second), func(t *testing.T) { + target := "http://fixture.invalid/queue" + hook := newDrainPollHook(target) + hook.inner = &drainPhysicalWriteTransport{first: first, second: second} + request, err := http.NewRequest(http.MethodGet, target, nil) + if err != nil { + t.Fatal(err) + } + firstResponse, _ := hook.RoundTrip(request) + if firstResponse != nil { + hook.releaseResponse() + _, _ = io.ReadAll(firstResponse.Body) + _ = firstResponse.Body.Close() + } + secondResponse, _ := hook.RoundTrip(request) + if secondResponse != nil { + _, _ = io.ReadAll(secondResponse.Body) + _ = secondResponse.Body.Close() + } + wantValid := first == drainWriteSuccess && second == drainWriteSuccess + if got := hook.pollWritesValid(); got != wantValid { + t.Fatalf("poll writes valid = %v, want %v; callbacks=%v invalid=%v", got, wantValid, hook.wroteCallbacks, hook.invalid) + } + for index, behavior := range []drainPhysicalWriteBehavior{first, second} { + want := 1 + if behavior == drainWriteDuplicate || behavior == drainWriteRetry { + want = 2 + } + if got := hook.wroteCallbacks[index+1]; got != want { + t.Fatalf("poll %d callbacks = %d, want %d", index+1, got, want) + } + } + }) + } + } +} + func TestDrainObservationRejectsPollCountersContradictingOwnedRunner(t *testing.T) { observation := validDrainTestObservation() // A valid internal partition with zero registered runners still contradicts diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index 0147f9a..669c568 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -95,26 +95,69 @@ type state struct { sessionID string reserved, uncertain, deleted bool phaseSeen map[string]bool + drainPhasePending bool + drainPhaseSequence int + drainPhaseSetID int observedJobs map[int64]bool workObserved bool inventory string } +func replayEventSequence(e Event, index int) int { + if e.Sequence > 0 { + return e.Sequence + } + return index + 1 +} + +func validDrainObservationForPhase(e Event, sequence int, s state) bool { + if !s.drainPhasePending || e.Drain == nil || !validDrainObservation(e.Drain) || e.Drain.Outcome != drainOutcomeObserved { + return false + } + return sequence > s.drainPhaseSequence && e.Drain.Sequence == s.drainPhaseSequence && s.setID > 0 && s.drainPhaseSetID == s.setID && e.Drain.Before.Set.ID == s.setID && e.Drain.After.Set.ID == s.setID +} + func replay(events []Event) state { s := state{phaseSeen: make(map[string]bool), observedJobs: make(map[int64]bool)} pending := "" - for _, e := range events { + for index, e := range events { switch e.Kind { case "baseline": // This library slice never grants a legacy fault/cleanup phase. s.reserved, s.uncertain, s.workObserved = true, true, true case "phase": - s.phaseSeen[e.Operation] = true if e.Operation == "drain" { - // The phase record is durable intent to observe a live session. - // A crash before its final observation must not reopen cleanup. + if s.phaseSeen[e.Operation] || s.drainPhasePending { + // A drain observation may discharge exactly one phase. A + // repeated phase is not collapsed into the same boolean fence. + s.uncertain = true + s.drainPhasePending = false + s.drainPhaseSequence = 0 + s.drainPhaseSetID = 0 + continue + } + s.drainPhasePending = true + s.drainPhaseSequence = replayEventSequence(e, index) + s.drainPhaseSetID = 0 + if e.ID > 0 { + if s.setID <= 0 || e.ID != s.setID { + s.uncertain = true + } + s.drainPhaseSetID = e.ID + } else { + // A drain phase must carry its created SetID. Never infer a + // missing, zero, or negative identity from prior state. + s.uncertain = true + } + } else if s.drainPhasePending { + // A different phase interrupts the active drain phase. Keep + // that history fenced even if a later record looks complete. s.uncertain = true + s.drainPhasePending = false + s.drainPhaseSequence = 0 + s.drainPhaseSetID = 0 } + s.phaseSeen[e.Operation] = true case "inventory": s.inventory = e.Digest case "observation": @@ -129,9 +172,18 @@ func replay(events []Event) state { } if e.Operation == "drain" { s.workObserved = true - if e.Drain == nil || e.Drain.Outcome != drainOutcomeObserved { + sequence := replayEventSequence(e, index) + if !validDrainObservationForPhase(e, sequence, s) { s.uncertain = true } + if s.drainPhasePending { + // A malformed or inconclusive record consumes the active + // phase without discharging it; a later duplicate cannot + // turn the same phase into an observed result. + s.drainPhasePending = false + s.drainPhaseSequence = 0 + s.drainPhaseSetID = 0 + } } if e.Operation == "drain-marker" { s.uncertain = true @@ -179,9 +231,14 @@ func replay(events []Event) state { case "unknown": s.uncertain = true pending = "" + if s.drainPhasePending { + s.drainPhasePending = false + s.drainPhaseSequence = 0 + s.drainPhaseSetID = 0 + } } } - s.uncertain = s.uncertain || pending != "" || s.sessionID != "" + s.uncertain = s.uncertain || pending != "" || s.sessionID != "" || s.drainPhasePending return s } @@ -297,7 +354,14 @@ func (d *Driver) Run(ctx context.Context, phase string) error { if phase == "inspect" { return d.inspect(ctx, s) } - if err := d.record(Event{Kind: "phase", Operation: phase}); err != nil { + phaseEvent := Event{Kind: "phase", Operation: phase} + if phase == "drain" { + // The phase carries the already-created set identity. The journal + // sequence assigned to this event is the only sequence a later drain + // observation may use to discharge its phase-local fence. + phaseEvent.ID = s.setID + } + if err := d.record(phaseEvent); err != nil { return err } if phase == "create" { diff --git a/experiments/g01-scaleset/livecanary/journal.go b/experiments/g01-scaleset/livecanary/journal.go index 19d9b1f..322c7ae 100644 --- a/experiments/g01-scaleset/livecanary/journal.go +++ b/experiments/g01-scaleset/livecanary/journal.go @@ -299,7 +299,7 @@ func validEvent(e Event) bool { case "authority": return e.Authority != nil case "phase": - return e.Operation != "" && e.Digest == "" + return e.Operation != "" && e.Digest == "" && (e.Operation != "drain" || e.ID > 0) case "inventory": return len(e.Digest) == 64 case "observation": diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index e7673e9..48bb7eb 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -291,12 +291,24 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { t.Fatalf("pinned SDK effects polls=%d ack=%d acquire=%d close=%d", polls, acks, acquires, closes) } var observed bool + var drainPhaseSequence, drainPhaseSetID, observationSequence int for _, event := range j.Events() { + if event.Kind == "phase" && event.Operation == "drain" { + drainPhaseSequence = event.Sequence + drainPhaseSetID = event.ID + } if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { observed = true + observationSequence = event.Drain.Sequence + if !sameDrainRunnerPartition(event.Drain.NextPoll.Statistics, event.Drain.Before.Statistics) { + t.Fatal("pinned SDK drain accepted a withdrawn-poll runner partition mismatch") + } } } if !observed { t.Fatal("pinned SDK drain did not retain observed outcome") } + if drainPhaseSequence <= 0 || drainPhaseSetID != set.ID || observationSequence != drainPhaseSequence { + t.Fatalf("pinned SDK drain binding = phase sequence %d SetID %d, observation %d; want exact phase sequence and SetID %d", drainPhaseSequence, drainPhaseSetID, observationSequence, set.ID) + } } diff --git a/experiments/g01-scaleset/livecanary/security_review_extra_test.go b/experiments/g01-scaleset/livecanary/security_review_extra_test.go new file mode 100644 index 0000000..3ec4b5c --- /dev/null +++ b/experiments/g01-scaleset/livecanary/security_review_extra_test.go @@ -0,0 +1,169 @@ +package livecanary + +import ( + "errors" + "testing" +) + +func setObservedRunnerPartition(o *drainObservation, registered, busy, idle int) { + for _, poll := range []*drainPollObservation{&o.Poll, &o.NextPoll} { + poll.Statistics.Registered = registered + poll.Statistics.Busy = busy + poll.Statistics.Idle = idle + } + for _, snapshot := range []*drainSnapshot{&o.Before, &o.After} { + snapshot.Statistics.Registered = registered + snapshot.Statistics.Busy = busy + snapshot.Statistics.Idle = idle + } +} + +func TestSecurityReviewDrainRequiresOwnedIdleBeforeProof(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*drainObservation) + }{ + { + name: "two-runners", + mutate: func(o *drainObservation) { + setObservedRunnerPartition(o, 2, 0, 2) + }, + }, + { + name: "busy", + mutate: func(o *drainObservation) { + setObservedRunnerPartition(o, 1, 1, 0) + }, + }, + { + name: "missing-owned-identity", + mutate: func(o *drainObservation) { + o.Before.Runner = nil + o.After.Runner = nil + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + observation := validDrainTestObservation() + tc.mutate(&observation) + if validDrainObservation(&observation) { + t.Fatalf("observed drain with %s prerequisite was accepted", tc.name) + } + }) + } + + a := approval() + a.Phases = append(a.Phases, "drain") + directory := privateDir(t) + j, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix() { + if err := j.Append(event); err != nil { + j.Close() + t.Fatalf("append drain prefix: %v", err) + } + } + observation := validDrainTestObservation() + setObservedRunnerPartition(&observation, 2, 0, 2) + phase := j.Events()[len(j.Events())-1] + observation.Sequence = phase.Sequence + appendErr := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}) + if appendErr == nil { + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if state := replay(reopened.Events()); !state.uncertain { + t.Fatalf("real FileJournal accepted non-owned idle proof and discharged fence: %+v", state) + } + return + } + if !errors.Is(appendErr, ErrJournal) { + j.Close() + t.Fatalf("non-owned idle observation append = %v, want ErrJournal", appendErr) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if events := reopened.Events(); len(events) != len(drainReplayPrefix()) { + t.Fatalf("reopened journal after rejected non-owned idle proof = %d events, want %d", len(events), len(drainReplayPrefix())) + } +} + +func TestSecurityReviewMatchingDrainPhaseDischargesAfterFileJournalReopen(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + directory := privateDir(t) + j, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix() { + if err := j.Append(event); err != nil { + j.Close() + t.Fatalf("append drain prefix: %v", err) + } + } + phase := j.Events()[len(j.Events())-1] + observation := validDrainTestObservation() + observation.Sequence = phase.Sequence + if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { + j.Close() + t.Fatalf("append valid drain observation: %v", err) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if state := replay(reopened.Events()); state.uncertain { + t.Fatalf("matching one-idle drain phase remained fenced after reopen: %+v", state) + } +} + +func TestSecurityReviewValidOneIdleRunnerDrainObservation(t *testing.T) { + if observation := validDrainTestObservation(); !validDrainObservation(&observation) { + t.Fatal("valid one-idle runner drain observation rejected") + } +} + +func TestSecurityReviewInconclusiveDrainMayHaveMissingOwnedIdentity(t *testing.T) { + observation := validDrainTestObservation() + observation.Outcome = drainOutcomeInconclusive + observation.Before.Runner = nil + observation.After.Runner = nil + if !validDrainObservation(&observation) { + t.Fatal("inconclusive drain with missing runner identity was rejected") + } +} + +func TestSecurityReviewObservedDrainAllowsLegitimateJobCounterChanges(t *testing.T) { + observation := validDrainTestObservation() + for _, poll := range []*drainPollObservation{&observation.Poll, &observation.NextPoll} { + poll.Statistics.Available = 4 + poll.Statistics.Acquired = 3 + poll.Statistics.Assigned = 2 + poll.Statistics.Running = 1 + } + observation.After.Statistics.Available = 4 + observation.After.Statistics.Acquired = 3 + observation.After.Statistics.Assigned = 2 + observation.After.Statistics.Running = 1 + if !validDrainObservation(&observation) { + t.Fatal("legitimate job-counter changes were rejected despite stable owned idle prerequisite") + } +} From f5020e8de32f8641e3aa80dfbc3e74e108277197 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 00:10:02 +0900 Subject: [PATCH 12/63] fix(g01): scope replay prerequisite to before snapshot --- docs/evidence/g01-idle-drain.md | 62 ++++++++++- .../livecanary/drain_followup_test.go | 105 ++++++++++++++++++ experiments/g01-scaleset/livecanary/driver.go | 65 ++++++++++- .../livecanary/sdk_integration_test.go | 38 ++++++- 4 files changed, 256 insertions(+), 14 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 5ddbdc3..7ec184a 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -174,6 +174,7 @@ run above is the new TDD regression, followed by the listed green runs. | Codex P1 [contradictory poll counters](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770561) | poll runner partitions must match the owned idle prerequisite before ACK/acquisition; `TestDrainObservationRejectsPollCountersContradictingOwnedRunner` and `TestDrainClientRejectsPollRunnerPartitionMismatchBeforeEffects` pass. | | Codex P1 [withdrawn-poll retry](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496285) | both bounded polls are traced; any second-poll error/duplicate/retry prevents `observed`, and the actual listener fixture remains inconclusive. `TestDrainListenerRejectsWithdrawnPollPhysicalRetry` passes. | | Codex P2 [drain-phase fence discharge](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496296) | replay discharges only a valid matching observed drain record's phase-local fence; unrelated uncertainty, work, reservations, and failed/inconclusive/crashed fences survive. `TestReplayDischargesCompletedDrainPhaseFence`, `TestReplayDrainFencePreservesUnrelatedUncertaintyAndReservations`, and `TestReplayDrainFenceRetainsInconclusiveOutcome` pass. | +| Codex P2 [after-snapshot prerequisite scope](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3969825423) | replay requires the first ordered drain snapshot to prove the owned idle prerequisite, then checks only after-snapshot identity and runner partition while allowing job-counter changes. The real pinned-SDK/FileJournal regression and malformed-stage matrix pass. | | Codex P1 [durable finding evidence](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496305) | this section and the matrix record full finding URLs, exact red/green commands, actual outcomes and rollback scope for the current and prior corrections. | | Security F1 / Protocol F1 poll reservation | `observe-poll` result journals one verified request ID plus fixed work before ACK; replay sets reservation/work fences. `TestDrainPollJournalsReservationBeforeACK` passes (`82d0d7d`). | | Security F2 rejected idle prerequisite | complete bounded `DrainSnapshot` plus `prerequisite-failed` marker is retained; replay remains uncertain. `TestDrainRejectedIdlePrerequisiteRetainsFence` passes (`82d0d7d`). | @@ -338,6 +339,58 @@ diff checks emitted no output. The pinned `github.com/actions/scaleset v0.4.0` Driver path remains offline-only, no broad multi-module gate was repeated, and no live operation or personal path was added. +### Codex r3969825423 phase-aware replay correction + +The fresh exact-head finding is [Codex r3969825423](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3969825423): replay was applying the before-only +`validDrainIdlePrerequisite` to the after `DrainSnapshot`, so a legitimate +`TotalAcquiredJobs=1` after snapshot permanently set uncertainty even though the +observed drain contract permits job-counter changes. The historical original +TDD exception and prior clean-history consolidation remain unchanged; this is a +new meaningful red-first regression for the current correction. + +Before the implementation change, the real pinned-SDK Driver/FileJournal +regression was run from `experiments/g01-scaleset`: + +```text +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestDriverDrainThroughPinnedSDKAndPollHook$' -count=1 -v +``` + +It failed after the listener completed and the FileJournal was closed/reopened: +`legitimate after job-counter change retained replay uncertainty` with +`uncertain:true`. The fixture recorded matching drain phase SetID/sequence, +before and after `DrainSnapshot` result records through the real Driver, and +kept the after runner partition/identity unchanged while changing only the +acquired-job counter. + +The minimal correction makes replay treat only the first ordered +`result/observe-runner` snapshot in a pending drain phase as the prerequisite; +the second must retain the created set identity, exact runner identity and +registered/busy/idle partition, while its job counters may change. Missing or +invalid before proof, a malformed after snapshot, or a snapshot outside that +phase-local ordering remains uncertain; unrelated reservations, work and +uncertainty are never cleared. + +The focused green, race, pinned-SDK, vet, formatting and diff checks were: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestDriverDrainThroughPinnedSDKAndPollHook|TestFileJournalReplayFencesMalformedDrainSnapshotStages|TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDrainObservedAllowsNextPollJobCounterChanges|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestDriverDrainThroughPinnedSDKAndPollHook|TestFileJournalReplayFencesMalformedDrainSnapshotStages|TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence|TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity|TestDrainObservedAllowsNextPollJobCounterChanges|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition' -count=1 +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +cd ../.. +bash scripts/gofmt.sh check +git diff --check +``` + +All listed checks passed: focused green, focused race, full `livecanary` +package, pinned SDK drain replay, vet, formatting and diff checks. The malformed +FileJournal matrix covers missing-before/no unsafe after inference, busy before, +after partition change, after scale-set identity change and after runner +identity change; each remains fenced. These are offline tests only; no live +runner, credential, workflow, session/JIT, cleanup or external operation was +performed. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an @@ -351,6 +404,9 @@ earlier correction must be isolated, revert only the reviewed source slice for [r3966770569](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770569), [r3966770561](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770561), or the [duplicate-key regression](https://github.com/1XP-AI/gh-runnerd/commit/6e1b2144cb92a1a53db3926dd0e924c646643dfa) -after checking dependent corrections; do not reset, erase or replay the -journal. No live cleanup, workflow replay, runner mutation or rollback -operation was performed. +after checking dependent corrections. To roll back only the current +[r3969825423](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3969825423) +correction, revert its focused source/test/evidence commit while retaining the +current journal and owned resources for inspection; do not reset, erase or +replay the journal. No live cleanup, workflow replay, runner mutation or +rollback operation was performed. diff --git a/experiments/g01-scaleset/livecanary/drain_followup_test.go b/experiments/g01-scaleset/livecanary/drain_followup_test.go index 3bee4d6..5315de7 100644 --- a/experiments/g01-scaleset/livecanary/drain_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_followup_test.go @@ -179,6 +179,111 @@ func TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity(t *testing.T } } +func appendDrainSnapshotResults(t *testing.T, j *FileJournal, snapshot drainSnapshot) { + t.Helper() + for _, event := range []Event{ + {Kind: "intent", Operation: "observe-owned"}, + {Kind: "result", Operation: "observe-owned", ID: snapshot.Set.ID}, + {Kind: "intent", Operation: "observe-runner"}, + {Kind: "result", Operation: "observe-runner", ID: snapshot.Runner.ID, DrainSnapshot: &snapshot}, + } { + if err := j.Append(event); err != nil { + t.Fatalf("append drain snapshot result: %v", err) + } + } +} + +func TestFileJournalReplayFencesMalformedDrainSnapshotStages(t *testing.T) { + for _, tc := range []struct { + name string + omitBefore bool + before, after func(*drainSnapshot) + }{ + { + name: "missing before does not infer after", + omitBefore: true, + after: func(snapshot *drainSnapshot) { + snapshot.Statistics.Acquired = 1 + }, + }, + { + name: "busy before", + before: func(snapshot *drainSnapshot) { + snapshot.Statistics.Busy = 1 + snapshot.Statistics.Idle = 0 + }, + }, + { + name: "after partition change", + after: func(snapshot *drainSnapshot) { + snapshot.Statistics.Registered = 2 + snapshot.Statistics.Idle = 2 + }, + }, + { + name: "after scale-set identity change", + after: func(snapshot *drainSnapshot) { + snapshot.Set.ID = 99 + snapshot.Runner.ScaleSetID = 99 + }, + }, + { + name: "after runner identity change", + after: func(snapshot *drainSnapshot) { + runner := *snapshot.Runner + runner.ID++ + snapshot.Runner = &runner + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + directory := privateDir(t) + j, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix() { + if err := j.Append(event); err != nil { + _ = j.Close() + t.Fatalf("append drain prefix: %v", err) + } + } + phase := j.Events()[len(j.Events())-1] + before := validDrainTestObservation().Before + after := validDrainTestObservation().After + if tc.before != nil { + tc.before(&before) + } + if tc.after != nil { + tc.after(&after) + } + if !tc.omitBefore { + appendDrainSnapshotResults(t, j, before) + } + appendDrainSnapshotResults(t, j, after) + observation := validDrainTestObservation() + observation.Sequence = phase.Sequence + if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { + _ = j.Close() + t.Fatalf("append final drain observation: %v", err) + } + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if state := replay(reopened.Events()); !state.uncertain { + t.Fatalf("malformed %s drain snapshots discharged replay fence: %+v", tc.name, state) + } + }) + } +} + func TestReplayDrainPhaseRequiresPositiveSetID(t *testing.T) { base := validDrainTestObservation() base.Sequence = 4 diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index 669c568..07e743c 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -98,6 +98,9 @@ type state struct { drainPhasePending bool drainPhaseSequence int drainPhaseSetID int + drainSnapshotCount int + drainBeforeSnapshot drainSnapshot + drainBeforeSnapshotValid bool observedJobs map[int64]bool workObserved bool inventory string @@ -117,6 +120,45 @@ func validDrainObservationForPhase(e Event, sequence int, s state) bool { return sequence > s.drainPhaseSequence && e.Drain.Sequence == s.drainPhaseSequence && s.setID > 0 && s.drainPhaseSetID == s.setID && e.Drain.Before.Set.ID == s.setID && e.Drain.After.Set.ID == s.setID } +func validDrainSnapshotForPhase(snapshot drainSnapshot, setID int, before *drainSnapshot) bool { + if setID <= 0 || !validDrainSnapshot(snapshot, snapshot.Set) || snapshot.Set.ID != setID { + return false + } + if before == nil { + return validDrainIdlePrerequisite(snapshot) + } + return before.Set == snapshot.Set && sameDrainRunner(before.Runner, snapshot.Runner) && sameDrainRunnerPartition(before.Statistics, snapshot.Statistics) +} + +func replayDrainSnapshot(e Event, index int, s *state) { + if e.DrainSnapshot == nil { + return + } + if e.Kind == "result" && e.Operation == "observe-runner" && s.drainPhasePending { + sequence := replayEventSequence(e, index) + switch s.drainSnapshotCount { + case 0: + s.drainSnapshotCount = 1 + s.drainBeforeSnapshot = *e.DrainSnapshot + s.drainBeforeSnapshotValid = sequence > s.drainPhaseSequence && validDrainSnapshotForPhase(*e.DrainSnapshot, s.drainPhaseSetID, nil) + if !s.drainBeforeSnapshotValid { + s.uncertain = true + } + case 1: + s.drainSnapshotCount = 2 + if !s.drainBeforeSnapshotValid || !validDrainSnapshotForPhase(*e.DrainSnapshot, s.drainPhaseSetID, &s.drainBeforeSnapshot) { + s.uncertain = true + } + default: + s.uncertain = true + } + return + } + if s.drainPhasePending || !validDrainIdlePrerequisite(*e.DrainSnapshot) { + s.uncertain = true + } +} + func replay(events []Event) state { s := state{phaseSeen: make(map[string]bool), observedJobs: make(map[int64]bool)} pending := "" @@ -134,11 +176,17 @@ func replay(events []Event) state { s.drainPhasePending = false s.drainPhaseSequence = 0 s.drainPhaseSetID = 0 + s.drainSnapshotCount = 0 + s.drainBeforeSnapshot = drainSnapshot{} + s.drainBeforeSnapshotValid = false continue } s.drainPhasePending = true s.drainPhaseSequence = replayEventSequence(e, index) s.drainPhaseSetID = 0 + s.drainSnapshotCount = 0 + s.drainBeforeSnapshot = drainSnapshot{} + s.drainBeforeSnapshotValid = false if e.ID > 0 { if s.setID <= 0 || e.ID != s.setID { s.uncertain = true @@ -156,6 +204,9 @@ func replay(events []Event) state { s.drainPhasePending = false s.drainPhaseSequence = 0 s.drainPhaseSetID = 0 + s.drainSnapshotCount = 0 + s.drainBeforeSnapshot = drainSnapshot{} + s.drainBeforeSnapshotValid = false } s.phaseSeen[e.Operation] = true case "inventory": @@ -183,14 +234,15 @@ func replay(events []Event) state { s.drainPhasePending = false s.drainPhaseSequence = 0 s.drainPhaseSetID = 0 + s.drainSnapshotCount = 0 + s.drainBeforeSnapshot = drainSnapshot{} + s.drainBeforeSnapshotValid = false } } if e.Operation == "drain-marker" { s.uncertain = true } - if e.DrainSnapshot != nil && !validDrainIdlePrerequisite(*e.DrainSnapshot) { - s.uncertain = true - } + replayDrainSnapshot(e, index, &s) case "intent": if pending != "" { s.uncertain = true @@ -215,9 +267,7 @@ func replay(events []Event) state { s.observedJobs[id] = true } } - if e.DrainSnapshot != nil && !validDrainIdlePrerequisite(*e.DrainSnapshot) { - s.uncertain = true - } + replayDrainSnapshot(e, index, &s) switch e.Operation { case "create": s.setID = e.ID @@ -235,6 +285,9 @@ func replay(events []Event) state { s.drainPhasePending = false s.drainPhaseSequence = 0 s.drainPhaseSetID = 0 + s.drainSnapshotCount = 0 + s.drainBeforeSnapshot = drainSnapshot{} + s.drainBeforeSnapshotValid = false } } } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 48bb7eb..98ca75c 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -186,7 +186,7 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, } - var polls, acks, acquires, closes int + var polls, acks, acquires, closes, snapshotReads int var server *httptest.Server server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { @@ -239,7 +239,14 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { closes++ w.WriteHeader(http.StatusNoContent) case strings.HasSuffix(r.URL.Path, "/runnerscalesets/7") && r.Method == http.MethodGet: - _ = json.NewEncoder(w).Encode(set) + snapshotReads++ + snapshot := *set + if snapshotReads == 2 { + stats := *set.Statistics + stats.TotalAcquiredJobs = 1 + snapshot.Statistics = &stats + } + _ = json.NewEncoder(w).Encode(&snapshot) case strings.HasSuffix(r.URL.Path, "/agents") && r.Method == http.MethodGet: _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []any{map[string]any{"id": 19, "name": a.workerName(), "runnerScaleSetId": 7}}}) default: @@ -282,13 +289,23 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { } base.drainClientFactory = buildAPI api := fixtureSDK{base} - j := &memoryJournal{events: []Event{{Kind: "phase", Operation: "create"}, {Kind: "intent", Operation: "create"}, {Kind: "result", Operation: "create", ID: 7}}} + directory := privateDir(t) + j, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + _ = j.Close() + t.Fatalf("append create prefix: %v", err) + } + } d := Driver{Approval: a, Journal: j, API: api} if err := d.Run(context.Background(), "drain"); err != nil { t.Fatalf("pinned SDK drain: %v", err) } - if polls != 2 || acks != 1 || acquires != 1 || closes != 1 { - t.Fatalf("pinned SDK effects polls=%d ack=%d acquire=%d close=%d", polls, acks, acquires, closes) + if polls != 2 || acks != 1 || acquires != 1 || closes != 1 || snapshotReads != 2 { + t.Fatalf("pinned SDK effects polls=%d ack=%d acquire=%d close=%d snapshots=%d", polls, acks, acquires, closes, snapshotReads) } var observed bool var drainPhaseSequence, drainPhaseSetID, observationSequence int @@ -311,4 +328,15 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { if drainPhaseSequence <= 0 || drainPhaseSetID != set.ID || observationSequence != drainPhaseSequence { t.Fatalf("pinned SDK drain binding = phase sequence %d SetID %d, observation %d; want exact phase sequence and SetID %d", drainPhaseSequence, drainPhaseSetID, observationSequence, set.ID) } + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + defer reopened.Close() + if state := replay(reopened.Events()); state.uncertain { + t.Fatalf("legitimate after job-counter change retained replay uncertainty: %+v", state) + } } From 1769da60bfbfb261f6e08d862465975e733cc176 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 04:40:53 +0900 Subject: [PATCH 13/63] fix(g01): bind replay to ordered drain snapshots --- docs/evidence/g01-idle-drain.md | 66 +++++ experiments/g01-scaleset/livecanary/drain.go | 17 ++ .../g01-scaleset/livecanary/drain_driver.go | 8 +- .../livecanary/drain_followup_test.go | 50 ++-- .../g01-scaleset/livecanary/drain_test.go | 24 +- experiments/g01-scaleset/livecanary/driver.go | 182 ++++++++----- .../g01-scaleset/livecanary/journal.go | 6 + .../g01-scaleset/livecanary/preparation.go | 2 +- .../livecanary/replay_contract_red_test.go | 255 ++++++++++++++++++ .../livecanary/sdk_integration_test.go | 9 +- .../livecanary/security_review_extra_test.go | 10 +- 11 files changed, 529 insertions(+), 100 deletions(-) create mode 100644 experiments/g01-scaleset/livecanary/replay_contract_red_test.go diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 7ec184a..01284f8 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -391,6 +391,72 @@ identity change; each remains fenced. These are offline tests only; no live runner, credential, workflow, session/JIT, cleanup or external operation was performed. +### PR72 residual replay-contract correction (security F1–F3 / protocol P1–P2) + +The exact-head security and protocol reviews of +`f5020e8de32f8641e3aa80dfbc3e74e108277197` identified residual +replay gaps: missing, single, out-of-order, or post-completion snapshot stages +could discharge a drain phase; the final observation was not correlated to the +durable before/after records; and same-ID foreign set/runner metadata was +accepted. The existing harness set `workObserved=true` for a drain observation, +so the demonstrated histories remained blocked from destructive cleanup, but +they incorrectly removed the phase-local uncertainty fence and could affect +later non-cleanup authorization. + +Meaningful red probes were added before the implementation change and run +against the current exact head with real `FileJournal` close/reopen boundaries: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestReplayContract' -count=1 -v +``` + +The run failed for missing before/after stages, an extra post-completion +snapshot, a final runner mismatch, and foreign metadata sharing the phase SetID. +The red assertions required uncertainty after reopen; the old implementation +returned `uncertain:false` for those malformed histories. + +The minimal correction records explicit `before`/`after` roles on the two +`observe-runner` results, requires exactly one valid ordered pair, and requires +the final observation to follow the after result and match both durable records +for set identity, runner tuple, and registered/busy/idle partition. Job +counters remain excluded from that equality, so legitimate after-snapshot +counter changes remain accepted. A completed or interrupted phase rejects any +later drain snapshot, and production replay derives the expected set/runner +identity from the current approval rather than trusting candidate metadata. + +The correction coverage includes complete histories and every crash prefix, +missing/extra/outside-phase stages, swapped/substituted runners, stable-set +ownership, malformed phase identity/sequence, unrelated reservations and +unknown/work fences, the real pinned-SDK/FileJournal path, and cleanup fencing. +The focused green and bounded race results are recorded below with the final +verification commands. Rollback is a focused revert of this correction's +source/test/evidence commit, retaining the current journal and owned resources +for inspection; do not reset, erase, replay, or run live cleanup. + +The bounded verification commands completed successfully after the correction: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestReplayContract|TestReplayDrain|TestFileJournal.*Drain|TestSecurityReview.*Drain|TestDrainObservedAllowsNextPollJobCounterChanges|TestDrainListenerRejectsContradictoryWithdrawnPollRunner|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestReplayContract|TestReplayDrain|TestFileJournal.*Drain|TestSecurityReview.*Drain|TestDrainObservedAllowsNextPollJobCounterChanges|TestDrainListenerRejectsContradictoryWithdrawnPollRunner|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +gofmt -l livecanary/baseline_journal.go livecanary/drain.go livecanary/drain_driver.go livecanary/drain_followup_test.go livecanary/drain_test.go livecanary/driver.go livecanary/journal.go livecanary/preparation.go livecanary/replay_contract_red_test.go livecanary/security_review_extra_test.go livecanary/sdk_integration_test.go +cd ../.. +git diff --check +bash scripts/gofmt.sh check +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 +cd ../.. +bash scripts/check-offline-experiments.sh +``` + +The focused normal suite, focused race suite, vet, formatting, diff, and final +full `livecanary` package run passed; race emitted no report and formatting/diff +checks emitted no diagnostics. The offline experiment check also passed for the +scaleset, livecanary, and liveworker modules. These remain offline fixture +checks only. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 3519b67..c8d96a9 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -151,6 +151,14 @@ type drainSetIdentity struct { Label string `json:"label"` } +// drainPhaseIdentity is the approved owner boundary derived from the current +// approval. Replay must compare durable snapshots to this identity instead of +// treating a candidate snapshot's own metadata as its expected authority. +type drainPhaseIdentity struct { + Set drainSetIdentity `json:"set"` + RunnerName string `json:"runner_name"` +} + type drainRunnerIdentity struct { ID int `json:"id"` Name string `json:"name"` @@ -183,6 +191,15 @@ type drainObservation struct { ObservedAt time.Time `json:"observed_at"` } +func approvedDrainPhaseIdentity(a Approval, setID int) drainPhaseIdentity { + name := a.setName() + return drainPhaseIdentity{Set: drainSetIdentity{ID: setID, Name: name, RunnerGroupID: a.RunnerGroupID, Label: name}, RunnerName: a.workerName()} +} + +func validDrainPhaseIdentity(p drainPhaseIdentity) bool { + return p.Set.ID > 0 && p.Set.RunnerGroupID > 0 && p.Set.Name != "" && p.Set.Label != "" && baselineText(p.Set.Name, 128) && baselineText(p.Set.Label, 128) && p.RunnerName != "" && baselineText(p.RunnerName, 256) +} + func validDrainResponse(value string) bool { return value == drainResponseNotAttempted || value == drainResponseSucceeded || value == drainResponseUnknown } diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index a8b1ccd..e81b446 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -151,13 +151,13 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr return Event{}, err } if runner == nil { - return Event{DrainSnapshot: &snapshot}, nil + return Event{DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, nil } if runner.ID <= 0 || runner.Name != d.Approval.workerName() || runner.RunnerScaleSetID != setID { - return Event{DrainSnapshot: &snapshot, Work: workUnresolved}, nil + return Event{DrainSnapshot: &snapshot, DrainSnapshotStage: stage, Work: workUnresolved}, nil } snapshot.Runner = &drainRunnerIdentity{ID: runner.ID, Name: runner.Name, ScaleSetID: runner.RunnerScaleSetID} - return Event{ID: runner.ID, DrainSnapshot: &snapshot}, nil + return Event{ID: runner.ID, DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, nil }); err != nil { return drainSnapshot{}, err } @@ -196,7 +196,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { if !ok { return ErrApproval } - phaseState := replay(d.Journal.Events()) + phaseState := replayWithApproval(d.Journal.Events(), &d.Approval) if !phaseState.drainPhasePending || phaseState.drainPhaseSequence <= 0 || phaseState.drainPhaseSetID != setID { return ErrQuarantine } diff --git a/experiments/g01-scaleset/livecanary/drain_followup_test.go b/experiments/g01-scaleset/livecanary/drain_followup_test.go index 5315de7..4923f36 100644 --- a/experiments/g01-scaleset/livecanary/drain_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_followup_test.go @@ -24,8 +24,18 @@ func drainReplayPrefix() []Event { } } +func appendDrainSnapshotReplayEvents(events []Event, snapshot drainSnapshot, stage string) []Event { + return append(events, + Event{Kind: "intent", Operation: "observe-owned"}, + Event{Kind: "result", Operation: "observe-owned", ID: snapshot.Set.ID}, + Event{Kind: "intent", Operation: "observe-runner"}, + Event{Kind: "result", Operation: "observe-runner", ID: snapshot.Runner.ID, DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, + ) +} + func TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence(t *testing.T) { - base := validDrainTestObservation() + a := approval() + base := drainObservationForApproval(a) base.Sequence = 4 foreign := base foreign.Before.Set.ID = 99 @@ -42,7 +52,9 @@ func TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence(t *testing.T) { name: "one matching phase", events: func(observation drainObservation) []Event { events := drainReplayPrefix() - events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}) + events = appendDrainSnapshotReplayEvents(events, observation.Before, "before") + events = appendDrainSnapshotReplayEvents(events, observation.After, "after") + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 13, Drain: &observation}) return events }, uncertain: false, @@ -115,8 +127,8 @@ func TestReplayDrainRequiresOneMatchingPhaseIdentityAndSequence(t *testing.T) { if tc.name == "mismatched created identity" { observation = foreign } - if got := replay(tc.events(observation)).uncertain; got != tc.uncertain { - t.Fatalf("uncertain=%v, want %v; state=%+v", got, tc.uncertain, replay(tc.events(observation))) + if got := replayWithApproval(tc.events(observation), &a).uncertain; got != tc.uncertain { + t.Fatalf("uncertain=%v, want %v; state=%+v", got, tc.uncertain, replayWithApproval(tc.events(observation), &a)) } }) } @@ -152,7 +164,7 @@ func TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity(t *testing.T t.Fatalf("append prefix: %v", err) } } - observation := validDrainTestObservation() + observation := drainObservationForApproval(a) if tc.name == "sequence" { observation.Sequence = 999 } else { @@ -172,20 +184,20 @@ func TestFileJournalDrainReplayRetainsMismatchedSequenceAndIdentity(t *testing.T t.Fatal(err) } defer reopened.Close() - if state := replay(reopened.Events()); !state.uncertain { + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { t.Fatalf("reopened mismatched drain history discharged fence: %+v", state) } }) } } -func appendDrainSnapshotResults(t *testing.T, j *FileJournal, snapshot drainSnapshot) { +func appendDrainSnapshotResults(t *testing.T, j *FileJournal, snapshot drainSnapshot, stage string) { t.Helper() for _, event := range []Event{ {Kind: "intent", Operation: "observe-owned"}, {Kind: "result", Operation: "observe-owned", ID: snapshot.Set.ID}, {Kind: "intent", Operation: "observe-runner"}, - {Kind: "result", Operation: "observe-runner", ID: snapshot.Runner.ID, DrainSnapshot: &snapshot}, + {Kind: "result", Operation: "observe-runner", ID: snapshot.Runner.ID, DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, } { if err := j.Append(event); err != nil { t.Fatalf("append drain snapshot result: %v", err) @@ -251,8 +263,9 @@ func TestFileJournalReplayFencesMalformedDrainSnapshotStages(t *testing.T) { } } phase := j.Events()[len(j.Events())-1] - before := validDrainTestObservation().Before - after := validDrainTestObservation().After + observation := drainObservationForApproval(a) + before := observation.Before + after := observation.After if tc.before != nil { tc.before(&before) } @@ -260,10 +273,10 @@ func TestFileJournalReplayFencesMalformedDrainSnapshotStages(t *testing.T) { tc.after(&after) } if !tc.omitBefore { - appendDrainSnapshotResults(t, j, before) + appendDrainSnapshotResults(t, j, before, "before") } - appendDrainSnapshotResults(t, j, after) - observation := validDrainTestObservation() + appendDrainSnapshotResults(t, j, after, "after") + observation = drainObservationForApproval(a) observation.Sequence = phase.Sequence if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { _ = j.Close() @@ -277,7 +290,7 @@ func TestFileJournalReplayFencesMalformedDrainSnapshotStages(t *testing.T) { t.Fatal(err) } defer reopened.Close() - if state := replay(reopened.Events()); !state.uncertain { + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { t.Fatalf("malformed %s drain snapshots discharged replay fence: %+v", tc.name, state) } }) @@ -285,7 +298,8 @@ func TestFileJournalReplayFencesMalformedDrainSnapshotStages(t *testing.T) { } func TestReplayDrainPhaseRequiresPositiveSetID(t *testing.T) { - base := validDrainTestObservation() + a := approval() + base := drainObservationForApproval(a) base.Sequence = 4 for _, tc := range []struct { name string @@ -300,7 +314,7 @@ func TestReplayDrainPhaseRequiresPositiveSetID(t *testing.T) { events := drainReplayPrefix() events[3].ID = tc.id events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &base}) - if state := replay(events); !state.uncertain { + if state := replayWithApproval(events, &a); !state.uncertain { t.Fatalf("drain phase with %d SetID discharged its fence: %+v", tc.id, state) } }) @@ -371,7 +385,7 @@ func TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen(t *testing.T) j.Close() t.Fatalf("append foreign drain phase: %v", err) } - observation := validDrainTestObservation() + observation := drainObservationForApproval(a) observation.Sequence = phase.Sequence if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { j.Close() @@ -385,7 +399,7 @@ func TestFileJournalForeignDrainPhaseSetIDRetainsFenceAfterReopen(t *testing.T) t.Fatal(err) } defer reopened.Close() - if state := replay(reopened.Events()); !state.uncertain { + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { t.Fatalf("reopened foreign drain phase discharged its fence: %+v", state) } } diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 5cf61c3..bfa543a 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -288,15 +288,19 @@ func TestDrainPhaseStartRetainsCrashUncertainty(t *testing.T) { } func TestReplayDischargesCompletedDrainPhaseFence(t *testing.T) { - observation := validDrainTestObservation() + a := approval() + observation := drainObservationForApproval(a) observation.Sequence = 4 - state := replay([]Event{ + events := []Event{ {Kind: "phase", Operation: "create", Sequence: 1}, {Kind: "intent", Operation: "create", Sequence: 2}, {Kind: "result", Operation: "create", Sequence: 3, ID: 7}, - {Kind: "phase", Operation: "drain", Sequence: 4, ID: 7}, - {Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}, - }) + } + events = append(events, drainReplayPrefix()[3]) + events = appendDrainSnapshotReplayEvents(events, observation.Before, "before") + events = appendDrainSnapshotReplayEvents(events, observation.After, "after") + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 13, Drain: &observation}) + state := replayWithApproval(events, &a) if state.uncertain { t.Fatalf("valid drain observation retained its completed phase fence: %+v", state) } @@ -933,6 +937,16 @@ func validDrainTestObservation() drainObservation { } } +func drainObservationForApproval(a Approval) drainObservation { + observation := validDrainTestObservation() + phase := approvedDrainPhaseIdentity(a, 7) + observation.Before.Set = phase.Set + observation.After.Set = phase.Set + observation.Before.Runner = &drainRunnerIdentity{ID: 19, Name: phase.RunnerName, ScaleSetID: phase.Set.ID} + observation.After.Runner = &drainRunnerIdentity{ID: 19, Name: phase.RunnerName, ScaleSetID: phase.Set.ID} + return observation +} + type drainDriverAPI struct { *fakeAPI approval Approval diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index 07e743c..0fd8827 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -41,21 +41,22 @@ type Approval struct { } type Event struct { - Baseline *baselineRecord `json:"baseline,omitempty"` - Drain *drainObservation `json:"drain,omitempty"` - DrainSnapshot *drainSnapshot `json:"drain_snapshot,omitempty"` - DrainMarker string `json:"drain_marker,omitempty"` - Authority *phaseAuthority `json:"authority,omitempty"` - Sequence int `json:"sequence"` - Kind string `json:"kind"` - Operation string `json:"operation,omitempty"` - ID int `json:"id,omitempty"` - SessionID string `json:"session_id,omitempty"` - RequestIDs []int64 `json:"request_ids,omitempty"` - Count int `json:"count,omitempty"` - Digest string `json:"digest,omitempty"` - Succeeded bool `json:"succeeded,omitempty"` - Work string `json:"work,omitempty"` + Baseline *baselineRecord `json:"baseline,omitempty"` + Drain *drainObservation `json:"drain,omitempty"` + DrainSnapshot *drainSnapshot `json:"drain_snapshot,omitempty"` + DrainMarker string `json:"drain_marker,omitempty"` + DrainSnapshotStage string `json:"drain_snapshot_stage,omitempty"` + Authority *phaseAuthority `json:"authority,omitempty"` + Sequence int `json:"sequence"` + Kind string `json:"kind"` + Operation string `json:"operation,omitempty"` + ID int `json:"id,omitempty"` + SessionID string `json:"session_id,omitempty"` + RequestIDs []int64 `json:"request_ids,omitempty"` + Count int `json:"count,omitempty"` + Digest string `json:"digest,omitempty"` + Succeeded bool `json:"succeeded,omitempty"` + Work string `json:"work,omitempty"` } type Journal interface { @@ -98,9 +99,15 @@ type state struct { drainPhasePending bool drainPhaseSequence int drainPhaseSetID int + drainPhase drainPhaseIdentity + drainPhaseIdentityValid bool + drainPhaseCompleted bool drainSnapshotCount int drainBeforeSnapshot drainSnapshot drainBeforeSnapshotValid bool + drainAfterSnapshot drainSnapshot + drainAfterSnapshotValid bool + drainAfterSnapshotSequence int observedJobs map[int64]bool workObserved bool inventory string @@ -113,15 +120,25 @@ func replayEventSequence(e Event, index int) int { return index + 1 } +func sameDrainSnapshotStable(left, right drainSnapshot) bool { + return left.Set == right.Set && sameDrainRunner(left.Runner, right.Runner) && sameDrainRunnerPartition(left.Statistics, right.Statistics) +} + func validDrainObservationForPhase(e Event, sequence int, s state) bool { - if !s.drainPhasePending || e.Drain == nil || !validDrainObservation(e.Drain) || e.Drain.Outcome != drainOutcomeObserved { + if !s.drainPhasePending || !s.drainPhaseIdentityValid || e.Drain == nil || !validDrainObservation(e.Drain) || e.Drain.Outcome != drainOutcomeObserved { + return false + } + if s.drainSnapshotCount != 2 || !s.drainBeforeSnapshotValid || !s.drainAfterSnapshotValid || sequence <= s.drainAfterSnapshotSequence { + return false + } + if e.Drain.Sequence != s.drainPhaseSequence || s.setID <= 0 || s.drainPhaseSetID != s.setID { return false } - return sequence > s.drainPhaseSequence && e.Drain.Sequence == s.drainPhaseSequence && s.setID > 0 && s.drainPhaseSetID == s.setID && e.Drain.Before.Set.ID == s.setID && e.Drain.After.Set.ID == s.setID + return validDrainSnapshotForPhase(e.Drain.Before, s.drainPhase, nil) && validDrainSnapshotForPhase(e.Drain.After, s.drainPhase, &s.drainBeforeSnapshot) && sameDrainSnapshotStable(e.Drain.Before, s.drainBeforeSnapshot) && sameDrainSnapshotStable(e.Drain.After, s.drainAfterSnapshot) } -func validDrainSnapshotForPhase(snapshot drainSnapshot, setID int, before *drainSnapshot) bool { - if setID <= 0 || !validDrainSnapshot(snapshot, snapshot.Set) || snapshot.Set.ID != setID { +func validDrainSnapshotForPhase(snapshot drainSnapshot, phase drainPhaseIdentity, before *drainSnapshot) bool { + if !validDrainSnapshotIdentity(snapshot, phase) { return false } if before == nil { @@ -130,36 +147,71 @@ func validDrainSnapshotForPhase(snapshot drainSnapshot, setID int, before *drain return before.Set == snapshot.Set && sameDrainRunner(before.Runner, snapshot.Runner) && sameDrainRunnerPartition(before.Statistics, snapshot.Statistics) } +func validDrainSnapshotIdentity(snapshot drainSnapshot, phase drainPhaseIdentity) bool { + return validDrainPhaseIdentity(phase) && validDrainSnapshot(snapshot, phase.Set) && snapshot.Runner != nil && snapshot.Runner.Name == phase.RunnerName && snapshot.Runner.ScaleSetID == phase.Set.ID +} + +func resetDrainPhase(s *state) { + s.drainPhasePending = false + s.drainPhaseSequence = 0 + s.drainPhaseSetID = 0 + s.drainPhase = drainPhaseIdentity{} + s.drainPhaseIdentityValid = false + s.drainSnapshotCount = 0 + s.drainBeforeSnapshot = drainSnapshot{} + s.drainBeforeSnapshotValid = false + s.drainAfterSnapshot = drainSnapshot{} + s.drainAfterSnapshotValid = false + s.drainAfterSnapshotSequence = 0 +} + func replayDrainSnapshot(e Event, index int, s *state) { if e.DrainSnapshot == nil { return } - if e.Kind == "result" && e.Operation == "observe-runner" && s.drainPhasePending { - sequence := replayEventSequence(e, index) - switch s.drainSnapshotCount { - case 0: - s.drainSnapshotCount = 1 - s.drainBeforeSnapshot = *e.DrainSnapshot - s.drainBeforeSnapshotValid = sequence > s.drainPhaseSequence && validDrainSnapshotForPhase(*e.DrainSnapshot, s.drainPhaseSetID, nil) - if !s.drainBeforeSnapshotValid { - s.uncertain = true - } - case 1: - s.drainSnapshotCount = 2 - if !s.drainBeforeSnapshotValid || !validDrainSnapshotForPhase(*e.DrainSnapshot, s.drainPhaseSetID, &s.drainBeforeSnapshot) { - s.uncertain = true - } - default: - s.uncertain = true - } + if e.Kind != "result" || e.Operation != "observe-runner" || !s.drainPhasePending || !s.drainPhaseIdentityValid { + // Drain snapshots are phase-local evidence. A snapshot before the + // phase, after its final observation, or after an interrupted phase + // must not be silently reused by a later replay. + s.uncertain = true + return + } + sequence := replayEventSequence(e, index) + if sequence <= s.drainPhaseSequence || !validDrainSnapshotIdentity(*e.DrainSnapshot, s.drainPhase) { + s.uncertain = true return } - if s.drainPhasePending || !validDrainIdlePrerequisite(*e.DrainSnapshot) { + switch e.DrainSnapshotStage { + case "before": + if s.drainSnapshotCount != 0 { + s.uncertain = true + return + } + s.drainSnapshotCount = 1 + s.drainBeforeSnapshot = *e.DrainSnapshot + s.drainBeforeSnapshotValid = validDrainSnapshotForPhase(*e.DrainSnapshot, s.drainPhase, nil) + if !s.drainBeforeSnapshotValid { + s.uncertain = true + } + case "after": + if s.drainSnapshotCount != 1 || !s.drainBeforeSnapshotValid || !validDrainSnapshotForPhase(*e.DrainSnapshot, s.drainPhase, &s.drainBeforeSnapshot) { + s.uncertain = true + return + } + s.drainSnapshotCount = 2 + s.drainAfterSnapshot = *e.DrainSnapshot + s.drainAfterSnapshotValid = true + s.drainAfterSnapshotSequence = sequence + default: s.uncertain = true } } func replay(events []Event) state { + return replayWithApproval(events, nil) +} + +func replayWithApproval(events []Event, approval *Approval) state { s := state{phaseSeen: make(map[string]bool), observedJobs: make(map[int64]bool)} pending := "" for index, e := range events { @@ -169,44 +221,48 @@ func replay(events []Event) state { s.reserved, s.uncertain, s.workObserved = true, true, true case "phase": if e.Operation == "drain" { - if s.phaseSeen[e.Operation] || s.drainPhasePending { + if s.phaseSeen[e.Operation] || s.drainPhasePending || s.drainPhaseCompleted { // A drain observation may discharge exactly one phase. A // repeated phase is not collapsed into the same boolean fence. s.uncertain = true - s.drainPhasePending = false - s.drainPhaseSequence = 0 - s.drainPhaseSetID = 0 - s.drainSnapshotCount = 0 - s.drainBeforeSnapshot = drainSnapshot{} - s.drainBeforeSnapshotValid = false + s.drainPhaseCompleted = true + resetDrainPhase(&s) continue } s.drainPhasePending = true + s.drainPhaseCompleted = false s.drainPhaseSequence = replayEventSequence(e, index) s.drainPhaseSetID = 0 + s.drainPhase = drainPhaseIdentity{} + s.drainPhaseIdentityValid = false s.drainSnapshotCount = 0 s.drainBeforeSnapshot = drainSnapshot{} s.drainBeforeSnapshotValid = false - if e.ID > 0 { + s.drainAfterSnapshot = drainSnapshot{} + s.drainAfterSnapshotValid = false + s.drainAfterSnapshotSequence = 0 + if e.ID > 0 && approval != nil { + phaseIdentity := approvedDrainPhaseIdentity(*approval, e.ID) + if !validDrainPhaseIdentity(phaseIdentity) { + s.uncertain = true + } if s.setID <= 0 || e.ID != s.setID { s.uncertain = true } s.drainPhaseSetID = e.ID + s.drainPhase = phaseIdentity + s.drainPhaseIdentityValid = validDrainPhaseIdentity(phaseIdentity) } else { - // A drain phase must carry its created SetID. Never infer a - // missing, zero, or negative identity from prior state. + // A drain phase must have an approved owner context and carry + // its created SetID. Never infer either from later records. s.uncertain = true } } else if s.drainPhasePending { // A different phase interrupts the active drain phase. Keep // that history fenced even if a later record looks complete. s.uncertain = true - s.drainPhasePending = false - s.drainPhaseSequence = 0 - s.drainPhaseSetID = 0 - s.drainSnapshotCount = 0 - s.drainBeforeSnapshot = drainSnapshot{} - s.drainBeforeSnapshotValid = false + s.drainPhaseCompleted = true + resetDrainPhase(&s) } s.phaseSeen[e.Operation] = true case "inventory": @@ -231,12 +287,8 @@ func replay(events []Event) state { // A malformed or inconclusive record consumes the active // phase without discharging it; a later duplicate cannot // turn the same phase into an observed result. - s.drainPhasePending = false - s.drainPhaseSequence = 0 - s.drainPhaseSetID = 0 - s.drainSnapshotCount = 0 - s.drainBeforeSnapshot = drainSnapshot{} - s.drainBeforeSnapshotValid = false + s.drainPhaseCompleted = true + resetDrainPhase(&s) } } if e.Operation == "drain-marker" { @@ -282,12 +334,8 @@ func replay(events []Event) state { s.uncertain = true pending = "" if s.drainPhasePending { - s.drainPhasePending = false - s.drainPhaseSequence = 0 - s.drainPhaseSetID = 0 - s.drainSnapshotCount = 0 - s.drainBeforeSnapshot = drainSnapshot{} - s.drainBeforeSnapshotValid = false + s.drainPhaseCompleted = true + resetDrainPhase(&s) } } } @@ -457,7 +505,7 @@ func (d *Driver) Run(ctx context.Context, phase string) error { return err } if phase == "cleanup" { - s = replay(d.Journal.Events()) // Includes the just-completed owned read. + s = replayWithApproval(d.Journal.Events(), &d.Approval) // Includes the just-completed owned read. // Aggregate zero alone never authorizes deletion. This scope has never // issued JIT/acquired a job, has no unresolved session, and must match // its original runner inventory as well as its immutable create receipt. diff --git a/experiments/g01-scaleset/livecanary/journal.go b/experiments/g01-scaleset/livecanary/journal.go index 322c7ae..f9653d8 100644 --- a/experiments/g01-scaleset/livecanary/journal.go +++ b/experiments/g01-scaleset/livecanary/journal.go @@ -285,6 +285,12 @@ func validEvent(e Event) bool { return false } } + if e.DrainSnapshotStage != "" && e.DrainSnapshotStage != "before" && e.DrainSnapshotStage != "after" { + return false + } + if e.DrainSnapshotStage != "" && e.DrainSnapshot == nil { + return false + } if e.DrainMarker != "" { if e.Kind != "observation" || e.Operation != "drain-marker" { return false diff --git a/experiments/g01-scaleset/livecanary/preparation.go b/experiments/g01-scaleset/livecanary/preparation.go index 0e34f01..1e6c120 100644 --- a/experiments/g01-scaleset/livecanary/preparation.go +++ b/experiments/g01-scaleset/livecanary/preparation.go @@ -24,7 +24,7 @@ func authorizePhase(a Approval, j Journal, phase string) (state, func(), error) return state{}, nil, ErrJournal } events := j.Events() - s := replay(events) + s := replayWithApproval(events, &a) invalid := phase != "inspect" && (s.uncertain || s.phaseSeen[phase] || s.deleted) // Run has not appended its phase record yet: an eligible create has no events. invalid = invalid || (phase == "create" && (s.setID != 0 || len(events) != 0)) diff --git a/experiments/g01-scaleset/livecanary/replay_contract_red_test.go b/experiments/g01-scaleset/livecanary/replay_contract_red_test.go new file mode 100644 index 0000000..8126d02 --- /dev/null +++ b/experiments/g01-scaleset/livecanary/replay_contract_red_test.go @@ -0,0 +1,255 @@ +package livecanary + +import "testing" + +func appendReplayContractSnapshot(t *testing.T, j *FileJournal, snapshot drainSnapshot, stage string) { + t.Helper() + for _, event := range []Event{ + {Kind: "intent", Operation: "observe-owned"}, + {Kind: "result", Operation: "observe-owned", ID: snapshot.Set.ID}, + {Kind: "intent", Operation: "observe-runner"}, + {Kind: "result", Operation: "observe-runner", ID: snapshot.Runner.ID, DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, + } { + if err := j.Append(event); err != nil { + t.Fatalf("append snapshot event: %v", err) + } + } +} + +func openReplayContractJournal(t *testing.T) (*FileJournal, Approval, Event) { + t.Helper() + a := approval() + a.Phases = append(a.Phases, "drain") + j, err := openTestJournal(t, privateDir(t), a) + if err != nil { + t.Fatal(err) + } + for _, event := range drainReplayPrefix() { + if err := j.Append(event); err != nil { + _ = j.Close() + t.Fatalf("append drain prefix: %v", err) + } + } + events := j.Events() + return j, a, events[len(events)-1] +} + +func reopenReplayContractJournal(t *testing.T, j *FileJournal, a Approval) *FileJournal { + t.Helper() + directory := j.directory + if err := j.Close(); err != nil { + t.Fatal(err) + } + reopened, err := openTestJournal(t, directory, a) + if err != nil { + t.Fatal(err) + } + return reopened +} + +func appendReplayContractObservation(t *testing.T, j *FileJournal, observation drainObservation, phase Event) { + t.Helper() + observation.Sequence = phase.Sequence + if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { + t.Fatalf("append drain observation: %v", err) + } +} + +func TestReplayContractRejectsMissingDrainSnapshotStagesAfterFileJournalReopen(t *testing.T) { + for _, tc := range []struct { + name string + appendBefore bool + appendAfter bool + }{ + {name: "missing both"}, + {name: "missing before", appendAfter: true}, + {name: "missing after", appendBefore: true}, + } { + t.Run(tc.name, func(t *testing.T) { + j, a, phase := openReplayContractJournal(t) + observation := drainObservationForApproval(a) + if tc.appendBefore { + appendReplayContractSnapshot(t, j, observation.Before, "before") + } + if tc.appendAfter { + appendReplayContractSnapshot(t, j, observation.After, "after") + } + appendReplayContractObservation(t, j, observation, phase) + reopened := reopenReplayContractJournal(t, j, a) + defer reopened.Close() + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { + t.Fatalf("accepted %s drain history without exactly two snapshots: %+v", tc.name, state) + } + }) + } +} + +func TestReplayContractRejectsExtraDrainSnapshotAfterCompletedPhase(t *testing.T) { + j, a, phase := openReplayContractJournal(t) + observation := drainObservationForApproval(a) + appendReplayContractSnapshot(t, j, observation.Before, "before") + appendReplayContractSnapshot(t, j, observation.After, "after") + appendReplayContractObservation(t, j, observation, phase) + appendReplayContractSnapshot(t, j, observation.After, "after") + reopened := reopenReplayContractJournal(t, j, a) + defer reopened.Close() + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { + t.Fatalf("accepted extra post-completion drain snapshot: %+v", state) + } +} + +func TestReplayContractRejectsFinalDrainSnapshotMismatch(t *testing.T) { + j, a, phase := openReplayContractJournal(t) + observed := drainObservationForApproval(a) + appendReplayContractSnapshot(t, j, observed.Before, "before") + appendReplayContractSnapshot(t, j, observed.After, "after") + observed.Before.Runner = &drainRunnerIdentity{ID: 20, Name: "g01-test-worker-2", ScaleSetID: 7} + observed.After.Runner = &drainRunnerIdentity{ID: 20, Name: "g01-test-worker-2", ScaleSetID: 7} + appendReplayContractObservation(t, j, observed, phase) + reopened := reopenReplayContractJournal(t, j, a) + defer reopened.Close() + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { + t.Fatalf("accepted final observation whose runner differs from durable snapshots: %+v", state) + } +} + +func TestReplayContractRejectsForeignMetadataWithMatchingSetID(t *testing.T) { + j, a, phase := openReplayContractJournal(t) + foreign := drainObservationForApproval(a).Before + foreign.Set.Name = "foreign-set" + foreign.Set.RunnerGroupID = 99 + foreign.Set.Label = "foreign-set" + foreign.Runner.Name = "foreign-worker" + appendReplayContractSnapshot(t, j, foreign, "before") + appendReplayContractSnapshot(t, j, foreign, "after") + observation := drainObservationForApproval(a) + observation.Before = foreign + observation.After = foreign + appendReplayContractObservation(t, j, observation, phase) + reopened := reopenReplayContractJournal(t, j, a) + defer reopened.Close() + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { + t.Fatalf("accepted foreign metadata with matching phase SetID: %+v", state) + } +} + +func TestReplayContractRequiresCompleteOrderedHistory(t *testing.T) { + a := approval() + observation := drainObservationForApproval(a) + observation.Sequence = 4 + events := drainReplayPrefix() + events = appendReplayContractSnapshotEvents(events, observation.Before, "before") + events = appendReplayContractSnapshotEvents(events, observation.After, "after") + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 13, Drain: &observation}) + for prefix := len(drainReplayPrefix()); prefix < len(events); prefix++ { + if state := replayWithApproval(events[:prefix], &a); !state.uncertain { + t.Fatalf("crash prefix %d discharged drain fence: %+v", prefix, state) + } + } + if state := replayWithApproval(events, &a); state.uncertain { + t.Fatalf("complete ordered drain history remained fenced: %+v", state) + } +} + +func appendReplayContractSnapshotEvents(events []Event, snapshot drainSnapshot, stage string) []Event { + return append(events, + Event{Kind: "intent", Operation: "observe-owned"}, + Event{Kind: "result", Operation: "observe-owned", ID: snapshot.Set.ID}, + Event{Kind: "intent", Operation: "observe-runner"}, + Event{Kind: "result", Operation: "observe-runner", ID: snapshot.Runner.ID, DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, + ) +} + +func TestReplayContractRejectsWrongSnapshotStageOrder(t *testing.T) { + a := approval() + observation := drainObservationForApproval(a) + cases := []struct { + name string + build func([]Event) []Event + }{ + { + name: "after before before", + build: func(events []Event) []Event { + events = appendReplayContractSnapshotEvents(events, observation.After, "after") + events = appendReplayContractSnapshotEvents(events, observation.Before, "before") + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 13, Drain: &observation}) + }, + }, + { + name: "final before after", + build: func(events []Event) []Event { + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 5, Drain: &observation}) + events = appendReplayContractSnapshotEvents(events, observation.Before, "before") + events = appendReplayContractSnapshotEvents(events, observation.After, "after") + return events + }, + }, + { + name: "duplicate before", + build: func(events []Event) []Event { + events = appendReplayContractSnapshotEvents(events, observation.Before, "before") + events = appendReplayContractSnapshotEvents(events, observation.Before, "before") + events = appendReplayContractSnapshotEvents(events, observation.After, "after") + return append(events, Event{Kind: "observation", Operation: "drain", Sequence: 17, Drain: &observation}) + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if state := replayWithApproval(tc.build(drainReplayPrefix()), &a); !state.uncertain { + t.Fatalf("accepted malformed ordered drain history: %+v", state) + } + }) + } +} + +func TestReplayContractRejectsSnapshotOutsideDrainPhase(t *testing.T) { + a := approval() + observation := drainObservationForApproval(a) + observation.Sequence = 4 + events := appendReplayContractSnapshotEvents(nil, observation.Before, "before") + events = append(events, drainReplayPrefix()...) + events = appendReplayContractSnapshotEvents(events, observation.After, "after") + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 17, Drain: &observation}) + if state := replayWithApproval(events, &a); !state.uncertain { + t.Fatalf("snapshot outside active drain phase discharged fence: %+v", state) + } +} + +func TestReplayWithApprovalBindsDrainIdentityToApprovedSet(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + snapshot := drainObservationForApproval(a).Before + observation := drainObservationForApproval(a) + observation.Sequence = 4 + events := []Event{ + {Kind: "phase", Operation: "create", Sequence: 1}, + {Kind: "intent", Operation: "create", Sequence: 2}, + {Kind: "result", Operation: "create", Sequence: 3, ID: 7}, + {Kind: "phase", Operation: "drain", Sequence: 4, ID: 7}, + } + events = appendReplayContractSnapshotEvents(events, snapshot, "before") + events = appendReplayContractSnapshotEvents(events, snapshot, "after") + events = append(events, Event{Kind: "observation", Operation: "drain", Sequence: 13, Drain: &observation}) + if state := replayWithApproval(events, &a); state.uncertain { + t.Fatalf("approved drain identity was not accepted: %+v", state) + } + foreign := snapshot + foreign.Set.Name = "foreign-set" + foreign.Set.Label = "foreign-set" + foreign.Set.RunnerGroupID = 99 + foreignRunnerName := "foreign-worker" + for i := 4; i < len(events); i++ { + if events[i].DrainSnapshot != nil { + snapshot := *events[i].DrainSnapshot + snapshot.Set = foreign.Set + runner := *snapshot.Runner + runner.Name = foreignRunnerName + snapshot.Runner = &runner + events[i].DrainSnapshot = &snapshot + } + } + if state := replayWithApproval(events, &a); !state.uncertain { + t.Fatalf("foreign snapshot identity discharged approved drain fence: %+v", state) + } +} diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 98ca75c..4672e87 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -309,11 +309,15 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { } var observed bool var drainPhaseSequence, drainPhaseSetID, observationSequence int + var snapshotStages []string for _, event := range j.Events() { if event.Kind == "phase" && event.Operation == "drain" { drainPhaseSequence = event.Sequence drainPhaseSetID = event.ID } + if event.DrainSnapshot != nil { + snapshotStages = append(snapshotStages, event.DrainSnapshotStage) + } if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { observed = true observationSequence = event.Drain.Sequence @@ -328,6 +332,9 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { if drainPhaseSequence <= 0 || drainPhaseSetID != set.ID || observationSequence != drainPhaseSequence { t.Fatalf("pinned SDK drain binding = phase sequence %d SetID %d, observation %d; want exact phase sequence and SetID %d", drainPhaseSequence, drainPhaseSetID, observationSequence, set.ID) } + if len(snapshotStages) != 2 || snapshotStages[0] != "before" || snapshotStages[1] != "after" { + t.Fatalf("pinned SDK drain snapshot stages = %v, want before/after", snapshotStages) + } if err := j.Close(); err != nil { t.Fatal(err) } @@ -336,7 +343,7 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { t.Fatal(err) } defer reopened.Close() - if state := replay(reopened.Events()); state.uncertain { + if state := replayWithApproval(reopened.Events(), &a); state.uncertain { t.Fatalf("legitimate after job-counter change retained replay uncertainty: %+v", state) } } diff --git a/experiments/g01-scaleset/livecanary/security_review_extra_test.go b/experiments/g01-scaleset/livecanary/security_review_extra_test.go index 3ec4b5c..14c6d1d 100644 --- a/experiments/g01-scaleset/livecanary/security_review_extra_test.go +++ b/experiments/g01-scaleset/livecanary/security_review_extra_test.go @@ -65,7 +65,7 @@ func TestSecurityReviewDrainRequiresOwnedIdleBeforeProof(t *testing.T) { t.Fatalf("append drain prefix: %v", err) } } - observation := validDrainTestObservation() + observation := drainObservationForApproval(a) setObservedRunnerPartition(&observation, 2, 0, 2) phase := j.Events()[len(j.Events())-1] observation.Sequence = phase.Sequence @@ -79,7 +79,7 @@ func TestSecurityReviewDrainRequiresOwnedIdleBeforeProof(t *testing.T) { t.Fatal(err) } defer reopened.Close() - if state := replay(reopened.Events()); !state.uncertain { + if state := replayWithApproval(reopened.Events(), &a); !state.uncertain { t.Fatalf("real FileJournal accepted non-owned idle proof and discharged fence: %+v", state) } return @@ -116,7 +116,9 @@ func TestSecurityReviewMatchingDrainPhaseDischargesAfterFileJournalReopen(t *tes } } phase := j.Events()[len(j.Events())-1] - observation := validDrainTestObservation() + observation := drainObservationForApproval(a) + appendDrainSnapshotResults(t, j, observation.Before, "before") + appendDrainSnapshotResults(t, j, observation.After, "after") observation.Sequence = phase.Sequence if err := j.Append(Event{Kind: "observation", Operation: "drain", Drain: &observation}); err != nil { j.Close() @@ -130,7 +132,7 @@ func TestSecurityReviewMatchingDrainPhaseDischargesAfterFileJournalReopen(t *tes t.Fatal(err) } defer reopened.Close() - if state := replay(reopened.Events()); state.uncertain { + if state := replayWithApproval(reopened.Events(), &a); state.uncertain { t.Fatalf("matching one-idle drain phase remained fenced after reopen: %+v", state) } } From 3a18028efec69eccfc40879a6f845c37d399e65f Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 04:48:18 +0900 Subject: [PATCH 14/63] fix(g01): enforce drain capacity ordinals --- docs/evidence/g01-idle-drain.md | 35 +++++++++++ experiments/g01-scaleset/livecanary/drain.go | 6 +- .../g01-scaleset/livecanary/drain_test.go | 58 +++++++++++++++++++ 3 files changed, 98 insertions(+), 1 deletion(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 01284f8..8fb779d 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -457,6 +457,41 @@ checks emitted no diagnostics. The offline experiment check also passed for the scaleset, livecanary, and liveworker modules. These remain offline fixture checks only. +### Codex r3972112659 capacity-ordinal correction + +The exact-head P1 finding is [Codex r3972112659](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972112659): `drainClient.GetMessage` accepted either capacity `0` or `1` for either of its two bounded calls. A malformed first poll could therefore reach the SDK with withdrawn capacity, and a malformed second poll could reach the SDK with capacity still set to `1`; the existing two-poll fence did not establish the required `1 -> 0` transition. + +The meaningful red regression was added before the implementation and run against the current exact head: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestDrainRejectsCapacityOrdinalBeforeInnerEffects$' -count=1 +``` + +It failed as expected: the first capacity-`0` call returned `` after reaching the inner session, and the rejected second capacity-`1` call left the inner poll count at `2` (with ACK/acquisition still at zero only because the later message fence stopped those effects). The regression controls also cover first capacity `-1` and `2`, a valid `1 -> 0` sequence, and a third call; rejected calls assert unchanged inner poll, ACK, and acquisition counts. + +The minimal green correction validates the ordinal under the existing client mutex before the inner SDK call: poll one must use capacity `1`, poll two must use capacity `0`, and any third poll or wrong capacity returns `ErrQuarantine` without advancing the ordinal or invoking the inner session. The existing pinned SDK/FileJournal test continues to exercise the legitimate `1 -> 0` HTTP header sequence, and no replay, persistence, cleanup, ownership, reservation, or unrelated fence behavior changed. + +Bounded verification completed successfully: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestDrainRejectsCapacityOrdinalBeforeInnerEffects|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestDrainRejectsCapacityOrdinalBeforeInnerEffects|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDriverDrainThroughPinnedSDKAndPollHook' -count=1 +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +gofmt -l livecanary/drain.go livecanary/drain_test.go +git diff --check +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 +``` + +The focused normal suite, focused race suite, pinned SDK path, vet, formatting, +diff, and one full `livecanary` package run passed; the formatting and diff +commands emitted no diagnostics. All checks were offline fixture tests; no live +runner, workflow, credential, cleanup, or external operation was performed. +Rollback is a focused revert of this correction's source/test/evidence commit, +retaining the current journal and owned resources for inspection; do not reset, +erase, replay, or run live cleanup. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index c8d96a9..86e615a 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -653,7 +653,11 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal return nil, c.rejectCall("observe-poll") } c.mu.Lock() - if capacity != drainInitialCapacity && capacity != drainWithdrawnCapacity || c.polls >= 2 { + wantCapacity := drainInitialCapacity + if c.polls == 1 { + wantCapacity = drainWithdrawnCapacity + } + if c.polls >= 2 || capacity != wantCapacity { c.mu.Unlock() return nil, ErrQuarantine } diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index bfa543a..5463573 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "net/http" "net/http/httptest" @@ -449,6 +450,60 @@ func TestDrainRejectsEffectsAfterCancellationAndRecordsMarker(t *testing.T) { t.Fatal("canceled drain did not persist cancellation marker") } +func TestDrainRejectsCapacityOrdinalBeforeInnerEffects(t *testing.T) { + newClient := func(message *scaleset.RunnerScaleSetMessage) (*drainClient, *drainGuardSession) { + inner := &drainGuardSession{ + initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: "fixture", Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}, + message: message, + } + obs := drainObservation{Poll: drainPollObservation{Message: drainMessageUnknown}, NextPoll: drainPollObservation{Message: drainMessageUnknown}} + return &drainClient{inner: inner, obs: &obs, phaseCtx: context.Background()}, inner + } + message := &scaleset.RunnerScaleSetMessage{ + MessageID: 7, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalAvailableJobs: 1, TotalAssignedJobs: 1, TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + JobAvailableMessages: []*scaleset.JobAvailable{{JobMessageBase: scaleset.JobMessageBase{RunnerRequestID: 11}}}, + } + + for _, capacity := range []int{drainWithdrawnCapacity, -1, 2} { + t.Run(fmt.Sprintf("first-capacity-%d", capacity), func(t *testing.T) { + client, inner := newClient(message) + if _, err := client.GetMessage(context.Background(), 0, capacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("first capacity %d error = %v, want quarantine", capacity, err) + } + if inner.pollCalls != 0 || inner.ackCalls != 0 || inner.acquireCalls != 0 { + t.Fatalf("first capacity %d performed effects: polls=%d ack=%d acquire=%d", capacity, inner.pollCalls, inner.ackCalls, inner.acquireCalls) + } + }) + } + + client, inner := newClient(message) + if _, err := client.GetMessage(context.Background(), 0, drainInitialCapacity); err != nil { + t.Fatalf("valid first poll: %v", err) + } + if _, err := client.GetMessage(context.Background(), 0, drainInitialCapacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("second capacity %d error = %v, want quarantine", drainInitialCapacity, err) + } + if inner.pollCalls != 1 || inner.ackCalls != 0 || inner.acquireCalls != 0 { + t.Fatalf("rejected second capacity performed effects: polls=%d ack=%d acquire=%d", inner.pollCalls, inner.ackCalls, inner.acquireCalls) + } + + client, inner = newClient(message) + if _, err := client.GetMessage(context.Background(), 0, drainInitialCapacity); err != nil { + t.Fatalf("third-call setup first poll: %v", err) + } + inner.message = nil + if _, err := client.GetMessage(context.Background(), 0, drainWithdrawnCapacity); err != nil { + t.Fatalf("valid second poll: %v", err) + } + if _, err := client.GetMessage(context.Background(), 0, drainWithdrawnCapacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("third poll error = %v, want quarantine", err) + } + if inner.pollCalls != 2 || inner.ackCalls != 0 || inner.acquireCalls != 0 { + t.Fatalf("rejected third poll performed effects: polls=%d ack=%d acquire=%d", inner.pollCalls, inner.ackCalls, inner.acquireCalls) + } +} + func TestDrainRejectsDuplicateOrWrongEffectsBeforeInnerCall(t *testing.T) { inner := &drainGuardSession{ initial: scaleset.RunnerScaleSetSession{SessionID: uuid.New(), OwnerName: "fixture", Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}}, @@ -1005,13 +1060,16 @@ func (a *drainDriverAPI) OpenDrainSession(_ context.Context, _ int, _ string, ho type drainGuardSession struct { initial scaleset.RunnerScaleSetSession message *scaleset.RunnerScaleSetMessage + pollCalls int ackCalls int acquireCalls int } func (s *drainGuardSession) Session() scaleset.RunnerScaleSetSession { return s.initial } func (s *drainGuardSession) Close(context.Context) error { return nil } + func (s *drainGuardSession) GetMessage(context.Context, int, int) (*scaleset.RunnerScaleSetMessage, error) { + s.pollCalls++ return s.message, nil } func (s *drainGuardSession) DeleteMessage(context.Context, int) error { From 668c361578c7cec749a650e452b85fd0ecf8e5ae Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 05:38:21 +0900 Subject: [PATCH 15/63] fix(g01): bind drain wire identity and poll cursor --- experiments/g01-scaleset/livecanary/drain.go | 50 +++- .../g01-scaleset/livecanary/drain_driver.go | 13 +- .../livecanary/drain_followup_test.go | 2 +- .../g01-scaleset/livecanary/drain_test.go | 15 +- experiments/g01-scaleset/livecanary/driver.go | 12 +- .../g01-scaleset/livecanary/journal.go | 2 +- .../livecanary/sdk_integration_test.go | 269 ++++++++++++++++++ .../livecanary/security_review_extra_test.go | 39 +++ 8 files changed, 384 insertions(+), 18 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 86e615a..d196e29 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -307,6 +307,8 @@ type drainPollHook struct { pollAttempts int statistics [3]drainStatistics statisticsKnown [3]bool + batches [3]*baselineBatch + batchesKnown [3]bool wroteCallbacks [3]int onRequestWritten func() wroteOnce sync.Once @@ -372,10 +374,12 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { response.Body = &drainObservedBody{ source: response.Body, status: response.StatusCode, - onComplete: func(stats drainStatistics, known bool) { + onComplete: func(stats drainStatistics, known bool, batch *baselineBatch, batchKnown bool) { h.mu.Lock() h.statistics[attempt] = stats h.statisticsKnown[attempt] = known + h.batches[attempt] = batch + h.batchesKnown[attempt] = batchKnown h.mu.Unlock() }, } @@ -408,11 +412,13 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { observedBody := &drainObservedBody{ source: response.Body, status: response.StatusCode, - onComplete: func(stats drainStatistics, known bool) { + onComplete: func(stats drainStatistics, known bool, batch *baselineBatch, batchKnown bool) { h.mu.Lock() if attempt >= 1 && attempt < len(h.statistics) { h.statistics[attempt] = stats h.statisticsKnown[attempt] = known + h.batches[attempt] = batch + h.batchesKnown[attempt] = batchKnown } h.mu.Unlock() }, @@ -467,9 +473,19 @@ func (h *drainPollHook) pollStatistics(index int) (drainStatistics, bool) { return h.statistics[index], h.statisticsKnown[index] } +func (h *drainPollHook) pollBatch(index int) (*baselineBatch, bool) { + if h == nil || index < 1 || index >= len(h.batches) { + return nil, false + } + h.mu.Lock() + defer h.mu.Unlock() + return h.batches[index], h.batchesKnown[index] +} + // drainObservedBody forwards response bytes unchanged while retaining a -// bounded, ephemeral copy solely to establish statistics field presence. It -// never persists, logs or rewrites the response payload. +// bounded, ephemeral copy solely to establish statistics field presence and +// strict embedded job facts. It never persists, logs or rewrites the response +// payload. type drainObservedBody struct { source io.ReadCloser status int @@ -477,7 +493,7 @@ type drainObservedBody struct { over bool mu sync.Mutex finished bool - onComplete func(drainStatistics, bool) + onComplete func(drainStatistics, bool, *baselineBatch, bool) } func (b *drainObservedBody) capture(data []byte) { @@ -509,16 +525,22 @@ func (b *drainObservedBody) finish() { b.mu.Unlock() stats, known := drainStatisticsFromBody(data) + batch, batchErr := decodeBaselineBatch(data) + batchKnown := batchErr == nil if b.status != http.StatusOK && b.status != http.StatusAccepted { known = false stats = drainStatistics{} + batch = nil + batchKnown = false } if over { known = false stats = drainStatistics{} + batch = nil + batchKnown = false } if b.onComplete != nil { - b.onComplete(stats, known) + b.onComplete(stats, known, batch, batchKnown) } } @@ -654,10 +676,12 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal } c.mu.Lock() wantCapacity := drainInitialCapacity + wantLast := 0 if c.polls == 1 { wantCapacity = drainWithdrawnCapacity + wantLast = c.messageID } - if c.polls >= 2 || capacity != wantCapacity { + if c.polls >= 2 || capacity != wantCapacity || last != wantLast || c.polls == 1 && (c.messageID <= 0 || c.obs.poll(c.polls).ACK != drainResponseSucceeded) { c.mu.Unlock() return nil, ErrQuarantine } @@ -713,6 +737,15 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal c.mu.Unlock() return nil, ErrQuarantine } + if c.hook != nil { + batch, batchKnown := c.hook.pollBatch(index) + if !batchKnown || !batch.matches(message) { + c.mu.Lock() + c.obs.poll(index).Message = drainMessageUnknown + c.mu.Unlock() + return nil, ErrQuarantine + } + } messageStats, statsErr := newDrainStatistics(message.Statistics) if statsErr != nil || (c.hook != nil && stats != messageStats) || (index <= 2 && c.ownedRunnerStatsKnown && !sameDrainRunnerPartition(stats, c.ownedRunnerStats)) || message.MessageID <= 0 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || len(message.JobAvailableMessages) != 1 || message.JobAvailableMessages[0] == nil || message.JobAvailableMessages[0].RunnerRequestID <= 0 { c.mu.Lock() @@ -805,11 +838,12 @@ func (s drainScaler) HandleDesiredRunnerCount(_ context.Context, count int) (int } s.client.mu.Lock() polls := s.client.polls + messageID := s.client.messageID s.client.mu.Unlock() if !s.client.active() { return 0, s.client.rejectCall("observe-poll") } - if polls >= 2 { + if polls >= 2 || polls >= 1 && messageID == 0 { return 0, errDrainCollected } return min(count, drainInitialCapacity), nil diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index e81b446..c8f9bbc 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -16,6 +16,7 @@ type journaledDrainClient struct { d *Driver inner Session sessionID string + hook *drainPollHook phaseCtx context.Context } @@ -58,6 +59,16 @@ func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity in if message == nil { return Event{SessionID: c.sessionID}, nil } + if c.hook != nil { + pollIndex := 1 + if capacity == drainWithdrawnCapacity { + pollIndex = 2 + } + batch, batchKnown := c.hook.pollBatch(pollIndex) + if !batchKnown || !batch.matches(message) { + return Event{}, ErrRemote + } + } if message.MessageID <= 0 || len(message.JobAvailableMessages) != 1 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || message.Statistics == nil { return Event{}, ErrRemote } @@ -241,7 +252,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return err } - journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID} + journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID, hook: hook} obs, runErr := runDrainListener(ctx, journaled, setID, hook) // The bounded observation sequence is phase-local: it must identify the // exact durable drain phase that was active when the listener ran. diff --git a/experiments/g01-scaleset/livecanary/drain_followup_test.go b/experiments/g01-scaleset/livecanary/drain_followup_test.go index 4923f36..f0b51bf 100644 --- a/experiments/g01-scaleset/livecanary/drain_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_followup_test.go @@ -436,7 +436,7 @@ func (t *contradictoryNextPollTransport) RoundTrip(req *http.Request) (*http.Res body := `{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}` status := http.StatusOK if t.polls == 1 { - body = `{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[]"}` + body = `{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[{\"messageType\":\"JobAvailable\",\"runnerRequestId\":11}]"}` } else { status = http.StatusAccepted body = `{"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":0,"totalBusyRunners":0,"totalIdleRunners":0}}` diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 5463573..2943a9d 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -37,7 +37,7 @@ func TestDrainListenerWithdrawsWhilePollResponseIsHeld(t *testing.T) { "totalBusyRunners": 0, "totalIdleRunners": 1, }, - "body": "[]", + "body": `[{"messageType":"JobAvailable","runnerRequestId":11}]`, } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { mu.Lock() @@ -492,14 +492,17 @@ func TestDrainRejectsCapacityOrdinalBeforeInnerEffects(t *testing.T) { if _, err := client.GetMessage(context.Background(), 0, drainInitialCapacity); err != nil { t.Fatalf("third-call setup first poll: %v", err) } + if err := client.DeleteMessage(context.Background(), 7); err != nil { + t.Fatalf("third-call setup ACK: %v", err) + } inner.message = nil - if _, err := client.GetMessage(context.Background(), 0, drainWithdrawnCapacity); err != nil { + if _, err := client.GetMessage(context.Background(), 7, drainWithdrawnCapacity); err != nil { t.Fatalf("valid second poll: %v", err) } if _, err := client.GetMessage(context.Background(), 0, drainWithdrawnCapacity); !errors.Is(err, ErrQuarantine) { t.Fatalf("third poll error = %v, want quarantine", err) } - if inner.pollCalls != 2 || inner.ackCalls != 0 || inner.acquireCalls != 0 { + if inner.pollCalls != 2 || inner.ackCalls != 1 || inner.acquireCalls != 0 { t.Fatalf("rejected third poll performed effects: polls=%d ack=%d acquire=%d", inner.pollCalls, inner.ackCalls, inner.acquireCalls) } } @@ -798,7 +801,7 @@ func (t *secondPollRetryTransport) RoundTrip(req *http.Request) (*http.Response, trace.WroteRequest(httptrace.WroteRequestInfo{}) return &http.Response{ StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[]"}`)), + Body: io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[{\"messageType\":\"JobAvailable\",\"runnerRequestId\":11}]"}`)), }, nil } if t.polls != 2 { @@ -1110,7 +1113,7 @@ func (t *drainNoTraceTransport) RoundTrip(req *http.Request) (*http.Response, er if t.gets == 1 { response.StatusCode = http.StatusOK response.Status = http.StatusText(http.StatusOK) - response.Body = io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalRegisteredRunners":1,"totalIdleRunners":1},"body":"[]"}`)) + response.Body = io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalRegisteredRunners":1,"totalIdleRunners":1},"body":"[{\"messageType\":\"JobAvailable\",\"runnerRequestId\":11}]"}`)) } else { response.StatusCode = http.StatusAccepted response.Status = http.StatusText(http.StatusAccepted) @@ -1172,7 +1175,7 @@ func (s *drainSyntheticSession) GetMessage(ctx context.Context, _, capacity int) return &scaleset.RunnerScaleSetMessage{ MessageID: 7, Statistics: &scaleset.RunnerScaleSetStatistic{TotalAvailableJobs: 1, TotalAssignedJobs: 1, TotalRegisteredRunners: 1, TotalIdleRunners: 1}, - JobAvailableMessages: []*scaleset.JobAvailable{{JobMessageBase: scaleset.JobMessageBase{RunnerRequestID: 11}}}, + JobAvailableMessages: []*scaleset.JobAvailable{{JobMessageBase: scaleset.JobMessageBase{JobMessageType: scaleset.JobMessageType{MessageType: scaleset.MessageTypeJobAvailable}, RunnerRequestID: 11}}}, }, nil } func (s *drainSyntheticSession) DeleteMessage(ctx context.Context, id int) error { diff --git a/experiments/g01-scaleset/livecanary/driver.go b/experiments/g01-scaleset/livecanary/driver.go index 0fd8827..f264a72 100644 --- a/experiments/g01-scaleset/livecanary/driver.go +++ b/experiments/g01-scaleset/livecanary/driver.go @@ -147,6 +147,16 @@ func validDrainSnapshotForPhase(snapshot drainSnapshot, phase drainPhaseIdentity return before.Set == snapshot.Set && sameDrainRunner(before.Runner, snapshot.Runner) && sameDrainRunnerPartition(before.Statistics, snapshot.Statistics) } +func validDrainSnapshotEventIdentity(e Event) bool { + if e.DrainSnapshot == nil { + return true + } + if e.DrainSnapshot.Runner == nil { + return e.ID == 0 + } + return e.ID > 0 && e.ID == e.DrainSnapshot.Runner.ID +} + func validDrainSnapshotIdentity(snapshot drainSnapshot, phase drainPhaseIdentity) bool { return validDrainPhaseIdentity(phase) && validDrainSnapshot(snapshot, phase.Set) && snapshot.Runner != nil && snapshot.Runner.Name == phase.RunnerName && snapshot.Runner.ScaleSetID == phase.Set.ID } @@ -169,7 +179,7 @@ func replayDrainSnapshot(e Event, index int, s *state) { if e.DrainSnapshot == nil { return } - if e.Kind != "result" || e.Operation != "observe-runner" || !s.drainPhasePending || !s.drainPhaseIdentityValid { + if !validDrainSnapshotEventIdentity(e) || e.Kind != "result" || e.Operation != "observe-runner" || !s.drainPhasePending || !s.drainPhaseIdentityValid { // Drain snapshots are phase-local evidence. A snapshot before the // phase, after its final observation, or after an interrupted phase // must not be silently reused by a later replay. diff --git a/experiments/g01-scaleset/livecanary/journal.go b/experiments/g01-scaleset/livecanary/journal.go index f9653d8..72f69e7 100644 --- a/experiments/g01-scaleset/livecanary/journal.go +++ b/experiments/g01-scaleset/livecanary/journal.go @@ -281,7 +281,7 @@ func validEvent(e Event) bool { } } if e.DrainSnapshot != nil { - if e.Kind != "result" || e.Operation != "observe-runner" || !validDrainSnapshot(*e.DrainSnapshot, e.DrainSnapshot.Set) { + if !validDrainSnapshotEventIdentity(e) || e.Kind != "result" || e.Operation != "observe-runner" || !validDrainSnapshot(*e.DrainSnapshot, e.DrainSnapshot.Set) { return false } } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 4672e87..ca58343 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -10,6 +10,8 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" + "sync/atomic" "testing" "time" @@ -24,6 +26,16 @@ func (f fixtureSDK) Preflight(context.Context, Approval) error { return n func (f fixtureSDK) Inventory(context.Context) (string, error) { return strings.Repeat("0", 64), nil } func (f fixtureSDK) VerifyRun(context.Context, Approval, int64) error { return nil } +type countingFixtureSDK struct { + fixtureSDK + verifyCalls atomic.Int32 +} + +func (f *countingFixtureSDK) VerifyRun(context.Context, Approval, int64) error { + f.verifyCalls.Add(1) + return nil +} + // Exercise the new live driver through the actual pinned SDK. This synthetic // transport is permanently fenced to its own loopback server, like the original // offline spike; no endpoint flag or environment can repoint it to GitHub. @@ -347,3 +359,260 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { t.Fatalf("legitimate after job-counter change retained replay uncertainty: %+v", state) } } + +func TestPinnedSDKDrainRejectsAmbiguousEmbeddedJobIdentityBeforeEffects(t *testing.T) { + a := approval() + for _, test := range []struct { + name string + body string + wantReject bool + }{ + // The pinned SDK's ordinary decoder accepts the case-folded duplicate + // and keeps the later value. The adapter must reject that ambiguous wire + // before the listener can ACK or acquire the message. + {name: "casefolded identity", body: `[{"messageType":"JobAvailable","runnerRequestId":41,"workflowRunId":5,"ownerName":"foreign-owner","OwnerName":"fixture-org","repositoryName":"canary"}]`, wantReject: true}, + {name: "duplicate identity", body: `[{"messageType":"JobAvailable","runnerRequestId":41,"workflowRunId":5,"ownerName":"fixture-org","ownerName":"fixture-org","repositoryName":"canary"}]`, wantReject: true}, + {name: "malformed body", body: `[{"messageType":"JobAvailable","runnerRequestId":41`, wantReject: true}, + {name: "legitimate SDK body", body: pinnedDrainJobBody(a)}, + } { + t.Run(test.name, func(t *testing.T) { + fixture, _, session, hook := newPinnedDrainSession(t, a, test.body) + observation, err := runDrainListener(context.Background(), session, 7, hook) + if test.wantReject { + if !errors.Is(err, ErrQuarantine) || observation.Outcome == drainOutcomeObserved { + t.Fatalf("ambiguous embedded job identity was accepted: observation=%+v err=%v", observation, err) + } + if fixture.acks.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("ambiguous embedded job identity reached effects: polls=%d acks=%d acquires=%d", fixture.polls.Load(), fixture.acks.Load(), fixture.acquires.Load()) + } + return + } + if err != nil || observation.Outcome != drainOutcomeObserved || fixture.acks.Load() != 1 || fixture.acquires.Load() != 1 { + t.Fatalf("legitimate embedded job path failed: observation=%+v err=%v polls=%d acks=%d acquires=%d", observation, err, fixture.polls.Load(), fixture.acks.Load(), fixture.acquires.Load()) + } + }) + } +} + +func TestPinnedSDKDrainBindsPollCursorBeforeInnerCall(t *testing.T) { + t.Run("first poll requires zero cursor", func(t *testing.T) { + a := approval() + fixture, _, session, hook := newPinnedDrainSession(t, a, pinnedDrainJobBody(a)) + c := newPinnedDrainClient(session, hook) + hook.releaseResponse() + + if _, err := c.GetMessage(context.Background(), 9, drainInitialCapacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("wrong first cursor = %v, want quarantine", err) + } + if fixture.polls.Load() != 0 { + t.Fatalf("wrong first cursor reached inner SDK: polls=%d", fixture.polls.Load()) + } + }) + + t.Run("second poll requires acknowledged message cursor", func(t *testing.T) { + a := approval() + fixture, _, session, hook := newPinnedDrainSession(t, a, pinnedDrainJobBody(a)) + c := newPinnedDrainClient(session, hook) + hook.releaseResponse() + + if _, err := c.GetMessage(context.Background(), 0, drainInitialCapacity); err != nil { + t.Fatalf("valid first poll: %v", err) + } + if _, err := c.GetMessage(context.Background(), 0, drainWithdrawnCapacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("wrong second cursor = %v, want quarantine", err) + } + if fixture.polls.Load() != 1 { + t.Fatalf("wrong second cursor reached inner SDK: polls=%d cursors=%v", fixture.polls.Load(), fixture.cursorsSnapshot()) + } + }) + + t.Run("legitimate SDK cursor path remains bounded", func(t *testing.T) { + a := approval() + fixture, _, session, hook := newPinnedDrainSession(t, a, pinnedDrainJobBody(a)) + c := newPinnedDrainClient(session, hook) + hook.releaseResponse() + + message, err := c.GetMessage(context.Background(), 0, drainInitialCapacity) + if err != nil || message == nil { + t.Fatalf("valid first poll = message %v err %v", message, err) + } + if err := c.DeleteMessage(context.Background(), message.MessageID); err != nil { + t.Fatalf("valid ACK: %v", err) + } + if _, err := c.AcquireJobs(context.Background(), []int64{41}); err != nil { + t.Fatalf("valid acquisition: %v", err) + } + if message, err = c.GetMessage(context.Background(), 17, drainWithdrawnCapacity); err != nil || message != nil { + t.Fatalf("valid second poll = message %v err %v", message, err) + } + if fixture.polls.Load() != 2 || fixture.acks.Load() != 1 || fixture.acquires.Load() != 1 { + t.Fatalf("legitimate SDK effects polls=%d acks=%d acquires=%d cursors=%v capacities=%v", fixture.polls.Load(), fixture.acks.Load(), fixture.acquires.Load(), fixture.cursorsSnapshot(), fixture.capacitiesSnapshot()) + } + }) +} + +func TestPinnedSDKDrainMatchesWireBeforeVerifyRun(t *testing.T) { + a := approval() + body := `[{"messageType":"JobAvailable","runnerRequestId":41,"workflowRunId":5,"ownerName":"foreign-owner","OwnerName":"fixture-org","repositoryName":"canary"}]` + fixture, base, session, hook := newPinnedDrainSession(t, a, body) + api := &countingFixtureSDK{fixtureSDK: fixtureSDK{base}} + d := &Driver{Approval: a, Journal: &memoryJournal{}, API: api} + c := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), hook: hook} + hook.releaseResponse() + + if _, err := c.GetMessage(context.Background(), 0, drainInitialCapacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("ambiguous embedded job identity = %v, want quarantine", err) + } + if api.verifyCalls.Load() != 0 { + t.Fatalf("wire mismatch reached VerifyRun: calls=%d", api.verifyCalls.Load()) + } + if fixture.acks.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("wire mismatch reached effects: acks=%d acquires=%d", fixture.acks.Load(), fixture.acquires.Load()) + } +} + +func pinnedDrainJobBody(a Approval) string { + data, _ := json.Marshal([]map[string]any{{ + "messageType": "JobAvailable", + "runnerRequestId": int64(41), + "workflowRunId": a.WorkflowRunID, + "ownerName": a.Organization, + "repositoryName": a.Repository, + }}) + return string(data) +} + +type pinnedDrainFixture struct { + server *httptest.Server + body string + polls atomic.Int32 + acks atomic.Int32 + acquires atomic.Int32 + mu sync.Mutex + cursors []string + capacity []string +} + +func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainFixture, *SDKAPI, Session, *drainPollHook) { + t.Helper() + fixture := &pinnedDrainFixture{body: body} + fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/actions/runners/registration-token") && r.Method == http.MethodPost: + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusCreated) + _ = json.NewEncoder(w).Encode(map[string]string{"token": "fixture-registration"}) + case strings.HasSuffix(r.URL.Path, "/actions/runner-registration") && r.Method == http.MethodPost: + claims, _ := json.Marshal(map[string]int64{"exp": time.Now().Add(time.Hour).Unix()}) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]string{ + "url": fixture.server.URL, + "token": "eyJhbGciOiJub25lIn0." + base64.RawURLEncoding.EncodeToString(claims) + ".", + }) + case strings.HasSuffix(r.URL.Path, "/sessions") && r.Method == http.MethodPost: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(scaleset.RunnerScaleSetSession{ + SessionID: uuid.MustParse("00000000-0000-4000-8000-000000000021"), OwnerName: a.setName(), + MessageQueueURL: fixture.server.URL + "/queue", MessageQueueAccessToken: "fixture-queue", + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }) + case r.URL.Path == "/queue" && r.Method == http.MethodGet: + fixture.polls.Add(1) + fixture.mu.Lock() + fixture.cursors = append(fixture.cursors, r.URL.Query().Get("lastMessageId")) + fixture.capacity = append(fixture.capacity, r.Header.Get(scaleset.HeaderScaleSetMaxCapacity)) + fixture.mu.Unlock() + if fixture.polls.Load() == 1 { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{ + "messageId": 17, "messageType": "RunnerScaleSetJobMessages", "body": fixture.body, + "statistics": pinnedDrainStatistics(1, 1), + }) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + _ = json.NewEncoder(w).Encode(map[string]any{"statistics": pinnedDrainStatistics(0, 0)}) + case r.URL.Path == "/queue/17" && r.Method == http.MethodDelete: + fixture.acks.Add(1) + w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/acquirejobs") && r.Method == http.MethodPost: + fixture.acquires.Add(1) + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []int64{41}}) + case strings.Contains(r.URL.Path, "/sessions/") && r.Method == http.MethodDelete: + w.WriteHeader(http.StatusNoContent) + default: + http.NotFound(w, r) + } + })) + t.Cleanup(fixture.server.Close) + + buildAPI := func(hook *drainPollHook) (*SDKAPI, error) { + retry := retryablehttp.NewClient() + retry.RetryMax = 0 + retry.Logger = nil + retry.HTTPClient.Timeout = time.Second + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.Proxy = nil + transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + if address != fixture.server.Listener.Addr().String() { + return nil, errors.New("non-fixture address denied") + } + return (&net.Dialer{}).DialContext(ctx, network, address) + } + wrappers := []func(http.RoundTripper) http.RoundTripper(nil) + if hook != nil { + wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { + hook.inner = inner + return hook + }) + } + retry.HTTPClient.Transport = withResponseBudget(transport, wrappers...) + options := []scaleset.HTTPOption{scaleset.WithRetryableHTTPClint(retry), scaleset.WithLogger(slog.New(slog.DiscardHandler))} + client, err := scaleset.NewClientWithPersonalAccessToken(scaleset.NewClientWithPersonalAccessTokenConfig{GitHubConfigURL: fixture.server.URL + "/fixture-org", PersonalAccessToken: "synthetic-installation"}, options...) + if err != nil { + return nil, err + } + return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: fixture.server.URL + "/api/v3", approval: a, options: options}, nil + } + base, err := buildAPI(nil) + if err != nil { + t.Fatal(err) + } + base.drainClientFactory = buildAPI + hook := newDrainPollHook("") + session, err := base.OpenDrainSession(context.Background(), 7, a.setName(), hook) + if err != nil { + t.Fatal(err) + } + return fixture, base, session, hook +} + +func newPinnedDrainClient(session Session, hook *drainPollHook) *drainClient { + initial := session.Session() + stats, _ := newDrainStatistics(initial.Statistics) + return &drainClient{ + inner: session, hook: hook, phaseCtx: context.Background(), + obs: &drainObservation{Poll: drainPollObservation{Message: drainMessageUnknown}, NextPoll: drainPollObservation{Message: drainMessageUnknown}}, ownedRunnerStats: stats, ownedRunnerStatsKnown: true, + } +} + +func pinnedDrainStatistics(available, assigned int) map[string]int { + return map[string]int{ + "totalAvailableJobs": available, "totalAcquiredJobs": 0, "totalAssignedJobs": assigned, + "totalRunningJobs": 0, "totalRegisteredRunners": 1, "totalBusyRunners": 0, "totalIdleRunners": 1, + } +} + +func (f *pinnedDrainFixture) cursorsSnapshot() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.cursors...) +} + +func (f *pinnedDrainFixture) capacitiesSnapshot() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.capacity...) +} diff --git a/experiments/g01-scaleset/livecanary/security_review_extra_test.go b/experiments/g01-scaleset/livecanary/security_review_extra_test.go index 14c6d1d..27855ea 100644 --- a/experiments/g01-scaleset/livecanary/security_review_extra_test.go +++ b/experiments/g01-scaleset/livecanary/security_review_extra_test.go @@ -169,3 +169,42 @@ func TestSecurityReviewObservedDrainAllowsLegitimateJobCounterChanges(t *testing t.Fatal("legitimate job-counter changes were rejected despite stable owned idle prerequisite") } } + +func TestSecurityReviewDrainSnapshotEventIDMustMatchRunner(t *testing.T) { + observation := validDrainTestObservation() + event := Event{ + Kind: "result", + Operation: "observe-runner", + ID: observation.Before.Runner.ID + 1, + DrainSnapshot: &observation.Before, + DrainSnapshotStage: "before", + } + if validEvent(event) { + t.Fatal("mismatched snapshot event ID was accepted") + } + + a := approval() + a.Phases = append(a.Phases, "drain") + j, err := openTestJournal(t, privateDir(t), a) + if err != nil { + t.Fatal(err) + } + defer j.Close() + for _, prefix := range drainReplayPrefix() { + if err := j.Append(prefix); err != nil { + t.Fatalf("append drain prefix: %v", err) + } + } + for _, prefix := range []Event{ + {Kind: "intent", Operation: "observe-owned"}, + {Kind: "result", Operation: "observe-owned", ID: observation.Before.Set.ID}, + {Kind: "intent", Operation: "observe-runner"}, + } { + if err := j.Append(prefix); err != nil { + t.Fatalf("append snapshot prefix: %v", err) + } + } + if err := j.Append(event); !errors.Is(err, ErrJournal) { + t.Fatalf("mismatched snapshot event ID append = %v, want ErrJournal", err) + } +} From 4d9043c456a957be2dd5db367f6d5d636483501a Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 05:41:46 +0900 Subject: [PATCH 16/63] docs(g01): record exact-head drain corrections --- docs/evidence/g01-idle-drain.md | 115 +++++++++++++++++++++++++++++--- 1 file changed, 105 insertions(+), 10 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 8fb779d..00dabb1 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -68,10 +68,11 @@ assertions: poll reservation omitted, rejected idle state not fencing replay, wrong ACK reaching the inner effect, no-message accepted as observed, and cancellation missing an explicit marker. Those failures are defect evidence, not a claim that the tests preceded implementation; the focused regressions -below pass on the current correction head. +below pass on the published correction snapshots. The newly queued f482 defects were first reproduced behaviorally on the -current f482 worktree with the added regressions: +historical f482 snapshot `f482d249e6e5eca7bd03ce55cdaf3b6cde7a671d` with the +added regressions: ```text cd experiments/g01-scaleset @@ -216,7 +217,23 @@ qualification. No credentials, runner/session/JIT operation, workflow operation, app/keychain/launchd/Docker/Lima mutation or cleanup was performed. -### Exact-head blocker corrections (local uncommitted tree) +### Historical exact-head blocker corrections (snapshot `1eb48afb...`; published correction `5d715c486...`) + +The following entries preserve the historical chronology of the blocker +corrections. The independent probes targeted source snapshot +`1eb48afb50ffbb10b42d07181f16153df1c494eb`; the corresponding correction was +published at full commit +`5d715c486c959ae67ca615c4eaea3e7e89ede556`, with subsequent replay and +capacity snapshots `f5020e8de32f8641e3aa80dfbc3e74e108277197`, +`1769da60bfbfb261f6e08d862465975e733cc176` and +`3a18028efec69eccfc40879a6f845c37d399e65f`. These entries identify historical +snapshots and do not describe the present branch. + +The approved TDD exception remains explicit: when chronological +pre-implementation execution cannot be recovered, independent test-only +probes may be added to an immutable archive snapshot to reproduce the old +behavior. Those retrospective reds remain evidence of that snapshot and are +not relabeled as pre-implementation tests. Before edits, the independent review probes were re-run against the immutable `1eb48afb50ffbb10b42d07181f16153df1c494eb` source extraction with: @@ -234,7 +251,7 @@ poll partition of `registered=0,busy=0,idle=0` to `observed`. The probes also confirmed the existing runner continuity, byte/body budget, retry trace and ACK-before-acquisition controls remained intact before this correction. -The local red-first correction adds phase-local replay binding and the +The historical red-first correction adds phase-local replay binding and the withdrawn-poll runner-partition fence. A drain phase records its created scale-set ID, and its journal-assigned event sequence becomes the only valid `Drain.Sequence`; replay requires exactly one active phase, the created set ID @@ -242,7 +259,7 @@ in both snapshots, and a later matching observation. Both polls and the final snapshot compare only `registered`, `busy` and `idle` runner counters, so job counters remain free to change. -The focused correction and positive controls were run from the experiment +The focused historical correction and positive controls were run from the experiment module: ```text @@ -259,10 +276,10 @@ bash scripts/check-offline-experiments.sh All commands passed. The first includes the real FileJournal close/reopen checks and the real pinned `github.com/actions/scaleset v0.4.0` listener path; the full package, focused race, vet, formatting and offline experiment checks -also passed. These remain offline tests only, and the local correction is -intentionally uncommitted and unpublished for coordinator exact-head review. +also passed. These remain offline tests only; that correction is published in +`5d715c486c959ae67ca615c4eaea3e7e89ede556`. -### Missing drain-phase SetID correction (current local uncommitted tree) +### Missing drain-phase SetID correction (historical snapshot; published in `5d715c486...`) The red-first regression was added before the implementation change and run with: @@ -299,7 +316,7 @@ no output. No broad multi-module gate was rerun for this narrow correction. These are offline tests only; no live runner, workflow, credential, journal cleanup, or external operation was performed. -### Security F2 owned-idle prerequisite correction (current local uncommitted tree) +### Security F2 owned-idle prerequisite correction (historical snapshot; published in `5d715c486...`) The prior F1 phase-ID correction remains in place; its separate red/green evidence above is unchanged and was included in the focused verification below. @@ -346,7 +363,7 @@ The fresh exact-head finding is [Codex r3969825423](https://github.com/1XP-AI/gh `TotalAcquiredJobs=1` after snapshot permanently set uncertainty even though the observed drain contract permits job-counter changes. The historical original TDD exception and prior clean-history consolidation remain unchanged; this is a -new meaningful red-first regression for the current correction. +new meaningful red-first regression for the then-reviewed correction snapshot. Before the implementation change, the real pinned-SDK Driver/FileJournal regression was run from `experiments/g01-scaleset`: @@ -492,6 +509,84 @@ Rollback is a focused revert of this correction's source/test/evidence commit, retaining the current journal and owned resources for inspection; do not reset, erase, replay, or run live cleanup. +### Exact-head follow-up: poll cursor and embedded wire identity + +Three new exact-head Codex findings were addressed from reviewed head +`3a18028efec69eccfc40879a6f845c37d399e65f`: + +* [P2 durable chronology](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972526860) + — the historical evidence text incorrectly described an obsolete local + tree. The historical section above now names the old source + snapshot and the published correction + `5d715c486c959ae67ca615c4eaea3e7e89ede556`, preserves the approved + retrospective test-only TDD exception, and contains no personal machine + path. +* [P1 poll cursor binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972526869) + — the first `GetMessage` must use `last=0`; the second must use the first + acknowledged message ID, and both checks occur before the inner SDK call. + A no-message first poll cannot create a fake future-cursor empty observation. +* [P1 embedded-body identity](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972526881) + — outer-envelope and JSON-encoded body fields are now decoded strictly, + including case-folded duplicate-key rejection, and bounded decoded job + facts must match the SDK message before `VerifyRun`, ACK or acquisition. + +The meaningful red was run first against the actual pinned +`github.com/actions/scaleset v0.4.0` loopback HTTP fixture: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestPinnedSDKDrain(RejectsAmbiguousEmbeddedJobIdentityBeforeEffects|BindsPollCursorBeforeInnerCall)' -count=1 -v +``` + +It exited 1 before the correction. The case-folded owner duplicate was +accepted as an observed message and reached ACK/acquisition; the wrong first +cursor and wrong second cursor both reached the inner SDK poll instead of +being quarantined. This red used atomic poll/ACK/acquisition counters in the +fixture, so it exercised the effect boundary rather than only a pure parser. + +The minimal correction uses the existing strict adapter parser at the +transport boundary. It retains only bounded structured `baselineBatch` facts +in the poll hook, clears the ephemeral body bytes, and journals no raw body or +SDK error. The journaled client compares those facts with the decoded SDK +message immediately after the inner poll returns and before `VerifyRun`; the +listener wrapper validates cursor/capacity/ACK state before invoking that inner +poll. Malformed, exact-duplicate, case-folded-duplicate and wrong-cursor +controls quarantine with zero forbidden effects, while the legitimate exact +SDK path still performs exactly one old poll, ACK, acquisition and withdrawn +poll. + +The read-only security archive also reported a same-name foreign numeric +runner ID that cannot be rejected without a trusted approved runtime ID. No +static ID was invented in this correction. The feasible persisted-evidence +gap was separately closed by requiring `Event.ID` to equal +`DrainSnapshot.Runner.ID`; the mismatch is covered by +`TestSecurityReviewDrainSnapshotEventIDMustMatchRunner` through a real +FileJournal append/rejection check. + +The source/test correction is commit +`668c361578c7cec749a650e452b85fd0ecf8e5ae`. Green verification completed as +follows: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrainRejectsAmbiguousEmbeddedJobIdentityBeforeEffects|TestPinnedSDKDrainBindsPollCursorBeforeInnerCall|TestPinnedSDKDrainMatchesWireBeforeVerifyRun|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDrainListenerRejectsWithdrawnPollPhysicalRetry|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition|TestDrainListenerNoMessageIsInconclusive|TestDrainRejectsCapacityOrdinalBeforeInnerEffects)$' -count=1 -v -timeout=180s +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrainRejectsAmbiguousEmbeddedJobIdentityBeforeEffects|TestPinnedSDKDrainBindsPollCursorBeforeInnerCall|TestPinnedSDKDrainMatchesWireBeforeVerifyRun|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDrainListenerRejectsWithdrawnPollPhysicalRetry|TestDrainListenerRejectsContradictoryWithdrawnPollRunnerPartition|TestDrainListenerNoMessageIsInconclusive|TestDrainRejectsCapacityOrdinalBeforeInnerEffects|TestSecurityReviewDrainSnapshotEventIDMustMatchRunner)$' -count=1 -v -timeout=180s +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +gofmt -l livecanary/drain.go livecanary/drain_driver.go livecanary/drain_followup_test.go livecanary/drain_test.go livecanary/driver.go livecanary/journal.go livecanary/sdk_integration_test.go livecanary/security_review_extra_test.go +git diff --check +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 +``` + +All listed commands passed; the race run emitted no report and formatting and +diff checks emitted no diagnostics. The full package run was the single broad +`livecanary` verification for this follow-up; all HTTP traffic stayed inside +the bounded loopback fixture and no live runner, workflow, credential, cleanup +or other external operation was performed. To roll back the source/test +correction, use `git revert --no-edit +668c361578c7cec749a650e452b85fd0ecf8e5ae` on the exact branch, retaining any +current journal and owned resources for inspection; do not reset, erase, +replay or run live cleanup. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an From 9b1f0c214022740d6276b6f9dd417f4f892d0bd5 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 06:34:32 +0900 Subject: [PATCH 17/63] fix(g01): enforce strict drain wire evidence --- docs/evidence/g01-idle-drain.md | 72 ++++- .../livecanary/baseline_listener.go | 2 +- .../livecanary/baseline_message.go | 38 ++- .../g01-scaleset/livecanary/baseline_wire.go | 19 ++ experiments/g01-scaleset/livecanary/drain.go | 171 +++++++++- .../g01-scaleset/livecanary/drain_driver.go | 60 +++- .../livecanary/drain_p1_followup_test.go | 292 ++++++++++++++++++ .../g01-scaleset/livecanary/drain_test.go | 26 +- .../g01-scaleset/livecanary/observer_http.go | 8 +- experiments/g01-scaleset/livecanary/sdk.go | 14 +- .../livecanary/sdk_integration_test.go | 125 +++++++- 11 files changed, 781 insertions(+), 46 deletions(-) create mode 100644 experiments/g01-scaleset/livecanary/drain_p1_followup_test.go diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 00dabb1..0fec7c3 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -229,11 +229,7 @@ capacity snapshots `f5020e8de32f8641e3aa80dfbc3e74e108277197`, `3a18028efec69eccfc40879a6f845c37d399e65f`. These entries identify historical snapshots and do not describe the present branch. -The approved TDD exception remains explicit: when chronological -pre-implementation execution cannot be recovered, independent test-only -probes may be added to an immutable archive snapshot to reproduce the old -behavior. Those retrospective reds remain evidence of that snapshot and are -not relabeled as pre-implementation tests. +The maintainer approval recorded at [Issue #71 comment](https://github.com/1XP-AI/gh-runnerd/issues/71#issuecomment-5603758575) is limited to the original Issue #71 chronology gap: because the original meaningful pre-implementation behavioral red could not be recovered, independent test-only probes may be run against immutable historical snapshots to diagnose that already-implemented behavior. This historical exception does not waive TDD for this follow-up or any future change: each new implementation correction still requires a meaningful failing test before the fix; retrospective archive reproductions are diagnostic evidence only and must not be presented as pre-implementation red or substituted for future TDD. Before edits, the independent review probes were re-run against the immutable `1eb48afb50ffbb10b42d07181f16153df1c494eb` source extraction with: @@ -518,8 +514,8 @@ Three new exact-head Codex findings were addressed from reviewed head — the historical evidence text incorrectly described an obsolete local tree. The historical section above now names the old source snapshot and the published correction - `5d715c486c959ae67ca615c4eaea3e7e89ede556`, preserves the approved - retrospective test-only TDD exception, and contains no personal machine + `5d715c486c959ae67ca615c4eaea3e7e89ede556`, preserves the initial Issue #71 + chronology exception described above, and contains no personal machine path. * [P1 poll cursor binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972526869) — the first `GetMessage` must use `last=0`; the second must use the first @@ -587,6 +583,68 @@ correction, use `git revert --no-edit current journal and owned resources for inspection; do not reset, erase, replay or run live cleanup. +### Exact-head follow-up: physical poll and strict remote facts + +The six latest exact-head Codex findings were reproduced from +`4d9043c456a957be2dd5db367f6d5d636483501a` before implementation and are +tracked at [physical poll capacity/header/cursor](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911079), [withdrawn 202 body](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911049), [cancel-before-release ordering](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911070), [strict VerifyRun fields](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911082), [strict acquirejobs count/value](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911056), and [strict scale-set snapshot facts](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911063). + +The meaningful red ran against the real pinned `github.com/actions/scaleset v0.4.0` loopback fixture before the corresponding source corrections: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrain(RejectsPhysicalPollMutationBeforeInner|RejectsWithdrawnPollBodyBeforeAbsent|RequiresCompletePollStatsBeforeVerifyRun|VerifyRunRejectsAmbiguousWireFieldsBeforeEffects|AcquisitionRequiresStrictWireResponse|SnapshotsRequireStrictWireFacts)$' -count=1 -v -timeout=180s +``` + +It exited 1 as intended. Physical header and cursor mutations were promoted +and reached the second poll; a withdrawn 202 carrying a complete message body +was classified as absent; incomplete outer statistics crossed `VerifyRun`; and +lossy SDK decoding accepted ambiguous exact/case-fold duplicate fields in +VerifyRun, acquisition, and snapshot responses (with the matrix also covering +malformed, null, missing, and contradictory values, some of which already +failed closed). The deterministic cancellation test was added red-first and +then fixed to require cancel before response release and listener join; no +remote effect is authorized by that helper. + +The correction inventories each evidence-bearing remote fact at its adapter +boundary and reuses the existing strict readers. Poll requests now compare the +SDK header and cursor against the callback ordinal and prior strict message ID, +using token-bearing query values only ephemerally; poll bodies distinguish an +unambiguous no-message wire shape from a lossy SDK nil; complete strict poll +statistics are checked before `VerifyRun`; VerifyRun, acquisition, and +scale-set snapshots compare strict bounded wire facts before any later effect. +An ambiguous post-acquire response records the effect as unknown through the +existing journal path, retains the owned reservation/uncertainty, and cannot +produce an observed drain. ACK-before-acquisition order, pinned SDK behavior, +and no raw credential/body/journal payload retention remain unchanged. + +Green verification for the source/test correction was: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner|TestDrainCancellationStopsBeforeReleasingHeldResponse|TestPinnedSDKDrainRejectsWithdrawnPollBodyBeforeAbsent|TestPinnedSDKDrainRequiresCompletePollStatsBeforeVerifyRun|TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects|TestPinnedSDKDrainAcquisitionRequiresStrictWireResponse|TestPinnedSDKDrainSnapshotsRequireStrictWireFacts|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDrainPollHookRequiresOneSuccessfulWritePerPoll|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=180s +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner|TestDrainCancellationStopsBeforeReleasingHeldResponse|TestPinnedSDKDrainRejectsWithdrawnPollBodyBeforeAbsent|TestPinnedSDKDrainRequiresCompletePollStatsBeforeVerifyRun|TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects|TestPinnedSDKDrainAcquisitionRequiresStrictWireResponse|TestPinnedSDKDrainSnapshotsRequireStrictWireFacts|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDrainPollHookRequiresOneSuccessfulWritePerPoll|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -timeout=180s +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +gofmt -l livecanary/drain.go livecanary/drain_driver.go livecanary/observer_http.go livecanary/sdk.go livecanary/baseline_message.go livecanary/baseline_wire.go livecanary/baseline_listener.go livecanary/drain_test.go livecanary/sdk_integration_test.go livecanary/drain_p1_followup_test.go +git diff --check +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 +``` + +The focused normal/race runs, vet, formatting, diff check, and one final full +`livecanary` run passed; race emitted no report and formatting/diff emitted no +diagnostics. The source/test correction is intentionally offline and makes no +live runner, workflow, credential, Docker/Lima, Keychain, launchd, cleanup or +GitHub write operation. Rollback is a focused `git revert --no-edit` of the +source/test correction (and its documentation commit, if separate), retaining +the current journal, owned reservation and uncertainty for inspection; never +reset, erase, replay or run live cleanup. Historical +`5d715c486c959ae67ca615c4eaea3e7e89ede556` and its later consolidated snapshots +`f5020e8de32f8641e3aa80dfbc3e74e108277197`, +`1769da60bfbfb261f6e08d862465975e733cc176`, and +`3a18028efec69eccfc40879a6f845c37d399e65f` remain unchanged, and this +evidence section contains no personal machine paths or raw secret-bearing +payloads. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an diff --git a/experiments/g01-scaleset/livecanary/baseline_listener.go b/experiments/g01-scaleset/livecanary/baseline_listener.go index 82c0e04..a674dc2 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener.go @@ -394,7 +394,7 @@ func (b *baselineListener) AcquireJobs(_ context.Context, ids []int64) ([]int64, _, _, accepted, status := c.facts() r.Accepted = accepted r.HTTPStatus = status - known := c.observed() && status == 200 && accepted != nil && accepted.Count != nil && *accepted.Count == 1 && len(accepted.IDs) == 1 && accepted.IDs[0] == ids[0] && ((callErr == nil && slices.Equal(got, ids)) || (callErr != nil && b.ctx.Err() != nil)) && b.session.Session().SessionID.String() == b.sessionID + known := c.observed() && status == 200 && accepted.matches(ids) && ((callErr == nil && slices.Equal(got, ids)) || (callErr != nil && b.ctx.Err() != nil)) && b.session.Session().SessionID.String() == b.sessionID resultRef, err := b.finish(r, known) if err != nil { return nil, err diff --git a/experiments/g01-scaleset/livecanary/baseline_message.go b/experiments/g01-scaleset/livecanary/baseline_message.go index cf30d74..db07b71 100644 --- a/experiments/g01-scaleset/livecanary/baseline_message.go +++ b/experiments/g01-scaleset/livecanary/baseline_message.go @@ -60,6 +60,11 @@ type baselineAccepted struct { Count *int `json:"count"` IDs []int64 `json:"ids"` } + +func (a *baselineAccepted) matches(ids []int64) bool { + return a != nil && a.Count != nil && *a.Count == len(ids) && slices.Equal(a.IDs, ids) +} + type baselineSetFacts struct { ID int `json:"id"` Name string `json:"name"` @@ -95,7 +100,27 @@ func decodeBaselineSet(data []byte) (*baselineSetFacts, error) { return &baselineSetFacts{w.ID, w.Name, w.Group, w.Labels, w.Setting.Disabled, s}, nil } func (s *baselineSetFacts) eligible(a Approval, id int) bool { - if s == nil || s.ID != id || s.Name != a.setName() || s.GroupID != a.RunnerGroupID || s.DisableUpdate == nil || !*s.DisableUpdate || !s.Statistics.eligible(false) { + return s.eligibleWithStats(a, id, false) +} + +func (s *baselineSetFacts) eligibleForDrain(a Approval, id int) bool { + if !s.matchesOwner(a, id) || s.Statistics == nil { + return false + } + for _, p := range []*int{s.Statistics.Available, s.Statistics.Acquired, s.Statistics.Assigned, s.Statistics.Running, s.Statistics.Registered, s.Statistics.Busy, s.Statistics.Idle} { + if p == nil || *p < 0 { + return false + } + } + return true +} + +func (s *baselineSetFacts) eligibleWithStats(a Approval, id int, acquired bool) bool { + return s.matchesOwner(a, id) && s.Statistics.eligible(acquired) +} + +func (s *baselineSetFacts) matchesOwner(a Approval, id int) bool { + if s == nil || s.ID != id || s.Name != a.setName() || s.GroupID != a.RunnerGroupID || s.DisableUpdate == nil || !*s.DisableUpdate { return false } for _, l := range s.Labels { @@ -105,6 +130,17 @@ func (s *baselineSetFacts) eligible(a Approval, id int) bool { } return false } + +// matches compares the bounded facts captured by the strict wire reader with +// the SDK value returned from the same request. A caller may use eligible to +// establish approved identity, but must also prove that the lossy SDK object +// did not disagree with those wire facts. +func (s *baselineSetFacts) matches(v *scaleset.RunnerScaleSet) bool { + if s == nil || v == nil || s.ID != v.ID || s.Name != v.Name || s.GroupID != v.RunnerGroupID || s.DisableUpdate == nil || *s.DisableUpdate != v.RunnerSetting.DisableUpdate || !slices.Equal(s.Labels, v.Labels) { + return false + } + return s.Statistics.matches(v.Statistics) +} func baselineText(s string, limit int) bool { if len(s) > limit || !utf8.ValidString(s) { return false diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 144ddfb..5219c37 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -69,6 +69,19 @@ func (c *baselineWireCapture) target(r *http.Request) bool { if c.stage == "session-open" { suffix += "sessions" } else if c.stage == "acquire" { + // The synthetic drain transport uses the queue URL for its bounded + // acquisition request. The released SDK uses the Actions API endpoint + // below; accepting this exact queue form keeps the strict adapter useful + // for both without widening the target matcher. + if c.queue != "" { + u, e := url.Parse(c.queue) + if e == nil && u.Scheme != "" && u.Host != "" && u.User == nil && u.Fragment == "" { + u.Path = strings.TrimSuffix(u.Path, "/") + "/acquirejobs" + if r.Method == "POST" && r.URL.String() == u.String() { + return true + } + } + } suffix += "acquirejobs" } else if c.stage == "jit" { suffix += "generatejitconfig" @@ -162,6 +175,12 @@ func (c *baselineWireCapture) facts() (*baselineSessionFacts, *baselineBatch, *b return c.session, c.batch, c.accepted, c.status } +func (c *baselineWireCapture) setFacts() (*baselineSetFacts, int) { + c.mu.Lock() + defer c.mu.Unlock() + return c.set, c.status +} + func validJITSecret(secret string) bool { if len(secret) < 16 || len(secret) > 1<<20 { return false diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index d196e29..7a239a4 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -10,6 +10,8 @@ import ( "net/http/httptrace" "net/url" "slices" + "strconv" + "strings" "sync" "time" @@ -144,6 +146,34 @@ func drainStatisticsFromBody(body []byte) (drainStatistics, bool) { return stats, true } +// drainMessageWireState classifies only the bounded response shape. An empty +// 202 body is the SDK's ordinary no-message response; a JSON object containing +// only statistics is also an unambiguous no-message response. Any malformed, +// unknown or message-bearing shape is retained only as a boolean present +// marker, so a lossy SDK nil cannot promote it to drainMessageAbsent. +func drainMessageWireState(body []byte) (present, absent bool) { + trimmed := bytes.TrimSpace(body) + if len(trimmed) == 0 { + return false, true + } + if !uniqueKeys(json.NewDecoder(bytes.NewReader(trimmed))) { + return true, false + } + var envelope map[string]json.RawMessage + if json.Unmarshal(trimmed, &envelope) != nil || envelope == nil { + return true, false + } + if len(envelope) == 0 { + return false, false + } + for key := range envelope { + if key != "statistics" { + return true, false + } + } + return false, true +} + type drainSetIdentity struct { ID int `json:"id"` Name string `json:"name"` @@ -309,6 +339,8 @@ type drainPollHook struct { statisticsKnown [3]bool batches [3]*baselineBatch batchesKnown [3]bool + messagePresent [3]bool + messageAbsent [3]bool wroteCallbacks [3]int onRequestWritten func() wroteOnce sync.Once @@ -328,12 +360,90 @@ func (h *drainPollHook) matches(req *http.Request) bool { target := h.target h.mu.Unlock() want, err := url.Parse(target) - if err != nil || want.Scheme == "" || want.Host == "" { + if err != nil || want.Scheme == "" || want.Host == "" || want.User != nil || want.Fragment != "" || want.EscapedPath() != want.Path { return false } // Queue access tokens may be embedded in the URL. Compare only the - // transport destination/path; never copy the query into an observation. - return req.URL.Scheme == want.Scheme && req.URL.Host == want.Host && req.URL.Path == want.Path + // transport destination/path here; the exact query is checked ephemerally + // immediately before the inner transport call and never copied to evidence. + return req.URL.Scheme == want.Scheme && req.URL.Host == want.Host && req.URL.Path == want.Path && req.URL.User == nil && req.URL.Fragment == "" && req.URL.EscapedPath() == req.URL.Path +} + +// sameDrainQuery compares parsed query values without retaining the token +// bearing URL. It deliberately preserves duplicate values, since silently +// normalizing them would turn an ambiguous cursor into an apparently valid one. +func sameDrainQuery(left, right url.Values) bool { + if len(left) != len(right) { + return false + } + for key, values := range left { + if !slices.Equal(values, right[key]) { + return false + } + } + return true +} + +func (h *drainPollHook) pollRequestValid(req *http.Request, attempt int) bool { + if h == nil || req == nil || req.URL == nil || (attempt != 1 && attempt != 2) { + return false + } + h.mu.Lock() + target := h.target + messageID := 0 + if attempt == 2 && h.batches[1] != nil { + messageID = h.batches[1].MessageID + } + h.mu.Unlock() + want, err := url.Parse(target) + if err != nil || want.Scheme == "" || want.Host == "" || want.User != nil || want.Fragment != "" || want.EscapedPath() != want.Path || req.Method != http.MethodGet || req.URL.Scheme != want.Scheme || req.URL.Host != want.Host || req.URL.Path != want.Path || req.URL.User != nil || req.URL.Fragment != "" || req.URL.EscapedPath() != req.URL.Path { + return false + } + values := req.Header.Values(scaleset.HeaderScaleSetMaxCapacity) + wantCapacity := drainInitialCapacity + if attempt == 2 { + wantCapacity = drainWithdrawnCapacity + } + if len(values) != 1 || values[0] != strconv.Itoa(wantCapacity) { + return false + } + wantQuery, err := url.ParseQuery(want.RawQuery) + if err != nil { + return false + } + gotQuery, err := url.ParseQuery(req.URL.RawQuery) + if err != nil { + return false + } + for key := range wantQuery { + if strings.EqualFold(key, "lastMessageId") { + return false + } + } + for key := range gotQuery { + if strings.EqualFold(key, "lastMessageId") { + if attempt == 1 || key != "lastMessageId" { + return false + } + } + } + if attempt == 1 { + return sameDrainQuery(wantQuery, gotQuery) + } + if messageID <= 0 { + return false + } + wantQuery.Set("lastMessageId", strconv.Itoa(messageID)) + return sameDrainQuery(wantQuery, gotQuery) +} + +func (h *drainPollHook) rejectPoll() (*http.Response, error) { + h.mu.Lock() + h.invalid = true + h.mu.Unlock() + h.wroteOnce.Do(func() { close(h.wrote) }) + h.responseOnce.Do(func() { close(h.response) }) + return nil, ErrQuarantine } func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { @@ -348,6 +458,9 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { attempt := h.pollAttempts first := h.pollAttempts == 1 h.mu.Unlock() + if !h.pollRequestValid(req, attempt) { + return h.rejectPoll() + } if !first { h.mu.Lock() second := h.pollAttempts == 2 @@ -374,12 +487,14 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { response.Body = &drainObservedBody{ source: response.Body, status: response.StatusCode, - onComplete: func(stats drainStatistics, known bool, batch *baselineBatch, batchKnown bool) { + onComplete: func(stats drainStatistics, known bool, batch *baselineBatch, batchKnown bool, present bool, absent bool) { h.mu.Lock() h.statistics[attempt] = stats h.statisticsKnown[attempt] = known h.batches[attempt] = batch h.batchesKnown[attempt] = batchKnown + h.messagePresent[attempt] = present + h.messageAbsent[attempt] = absent h.mu.Unlock() }, } @@ -412,13 +527,15 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { observedBody := &drainObservedBody{ source: response.Body, status: response.StatusCode, - onComplete: func(stats drainStatistics, known bool, batch *baselineBatch, batchKnown bool) { + onComplete: func(stats drainStatistics, known bool, batch *baselineBatch, batchKnown bool, present bool, absent bool) { h.mu.Lock() if attempt >= 1 && attempt < len(h.statistics) { h.statistics[attempt] = stats h.statisticsKnown[attempt] = known h.batches[attempt] = batch h.batchesKnown[attempt] = batchKnown + h.messagePresent[attempt] = present + h.messageAbsent[attempt] = absent } h.mu.Unlock() }, @@ -482,6 +599,15 @@ func (h *drainPollHook) pollBatch(index int) (*baselineBatch, bool) { return h.batches[index], h.batchesKnown[index] } +func (h *drainPollHook) pollMessageState(index int) (present, absent bool) { + if h == nil || index < 1 || index >= len(h.messagePresent) { + return false, false + } + h.mu.Lock() + defer h.mu.Unlock() + return h.messagePresent[index], h.messageAbsent[index] +} + // drainObservedBody forwards response bytes unchanged while retaining a // bounded, ephemeral copy solely to establish statistics field presence and // strict embedded job facts. It never persists, logs or rewrites the response @@ -493,7 +619,7 @@ type drainObservedBody struct { over bool mu sync.Mutex finished bool - onComplete func(drainStatistics, bool, *baselineBatch, bool) + onComplete func(drainStatistics, bool, *baselineBatch, bool, bool, bool) } func (b *drainObservedBody) capture(data []byte) { @@ -527,20 +653,25 @@ func (b *drainObservedBody) finish() { stats, known := drainStatisticsFromBody(data) batch, batchErr := decodeBaselineBatch(data) batchKnown := batchErr == nil + present, absent := drainMessageWireState(data) if b.status != http.StatusOK && b.status != http.StatusAccepted { known = false stats = drainStatistics{} batch = nil batchKnown = false + present = false + absent = false } if over { known = false stats = drainStatistics{} batch = nil batchKnown = false + present = false + absent = false } if b.onComplete != nil { - b.onComplete(stats, known, batch, batchKnown) + b.onComplete(stats, known, batch, batchKnown, present, absent) } } @@ -716,6 +847,16 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal } if message == nil { c.mu.Lock() + if c.hook != nil { + present, absent := c.hook.pollMessageState(index) + if present || !absent { + c.obs.poll(index).Message = drainMessageUnknown + c.obs.poll(index).Statistics = drainStatistics{} + c.obs.poll(index).StatsKnown = false + c.mu.Unlock() + return nil, ErrQuarantine + } + } if statsKnown && c.ownedRunnerStatsKnown && !sameDrainRunnerPartition(stats, c.ownedRunnerStats) { c.obs.poll(index).Message = drainMessageUnknown c.obs.poll(index).Statistics = drainStatistics{} @@ -855,6 +996,18 @@ func (drainScaler) HandleJobCompleted(context.Context, *scaleset.JobCompleted) e return ErrQuarantine } +func cancelAndJoinDrain(stop, release func(), join func()) { + if stop != nil { + stop() + } + if release != nil { + release() + } + if join != nil { + join() + } +} + func runDrainListener(ctx context.Context, client listener.Client, setID int, hook *drainPollHook) (drainObservation, error) { obs := drainObservation{Version: drainObservationVersion, Outcome: drainOutcomeInconclusive, InitialCapacity: drainInitialCapacity, WithdrawnCapacity: drainWithdrawnCapacity, Boundary: drainBoundaryUnresolved, ServerReceipt: drainServerReceiptUnproven, Sequence: 1, ObservedAt: time.Now().UTC()} obs.Poll = drainPollObservation{Capacity: drainInitialCapacity, Message: drainMessageUnknown, ACK: drainResponseNotAttempted, Acquisition: drainResponseNotAttempted} @@ -893,9 +1046,7 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho runResult := make(chan error, 1) go func() { runResult <- l.Run(runCtx, drainScaler{client: c}) }() cancelAndJoin := func() { - hook.releaseResponse() - stop() - <-runResult + cancelAndJoinDrain(stop, hook.releaseResponse, func() { <-runResult }) } release := false diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index c8f9bbc..32f4bec 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -12,10 +12,15 @@ type drainSessionOpener interface { OpenDrainSession(context.Context, int, string, *drainPollHook) (Session, error) } +type drainScaleSetWireReader interface { + drainGetScaleSet(context.Context, int, *baselineWireCapture) (*scaleset.RunnerScaleSet, error) +} + type journaledDrainClient struct { d *Driver inner Session sessionID string + setID int hook *drainPollHook phaseCtx context.Context } @@ -59,6 +64,20 @@ func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity in if message == nil { return Event{SessionID: c.sessionID}, nil } + messageStats, statsErr := newDrainStatistics(message.Statistics) + if statsErr != nil { + return Event{}, ErrRemote + } + if c.hook != nil { + pollIndex := 1 + if capacity == drainWithdrawnCapacity { + pollIndex = 2 + } + wireStats, statsKnown := c.hook.pollStatistics(pollIndex) + if !statsKnown || wireStats != messageStats { + return Event{}, ErrRemote + } + } if c.hook != nil { pollIndex := 1 if capacity == drainWithdrawnCapacity { @@ -72,9 +91,6 @@ func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity in if message.MessageID <= 0 || len(message.JobAvailableMessages) != 1 || len(message.JobAssignedMessages) != 0 || len(message.JobStartedMessages) != 0 || len(message.JobCompletedMessages) != 0 || message.Statistics == nil { return Event{}, ErrRemote } - if _, callErr = newDrainStatistics(message.Statistics); callErr != nil { - return Event{}, ErrRemote - } for _, job := range message.JobAvailableMessages { if job == nil || job.RunnerRequestID <= 0 || job.WorkflowRunID != c.d.Approval.WorkflowRunID || job.OwnerName != c.d.Approval.Organization || job.RepositoryName != c.d.Approval.Repository { return Event{}, ErrApproval @@ -116,8 +132,31 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] var got []int64 err = c.d.effect(callCtx, "acquire", ids, func(call context.Context) (Event, error) { var err error + var wire *baselineWireCapture + if c.hook != nil { + setID := c.setID + if setID <= 0 { + if session := c.inner.Session(); session.RunnerScaleSet != nil { + setID = session.RunnerScaleSet.ID + } + } + c.hook.mu.Lock() + queue := c.hook.target + c.hook.mu.Unlock() + wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue} + call = wire.context(call) + } got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) - if err != nil || !slices.Equal(got, ids) { + if err != nil { + return Event{}, errors.New("acquisition response did not match the one-shot request") + } + if wire != nil { + _, _, accepted, status := wire.facts() + if !wire.observed() || status != 200 || !accepted.matches(ids) { + return Event{}, errors.New("acquisition response did not match the one-shot request") + } + } + if !slices.Equal(got, ids) { return Event{}, errors.New("acquisition response did not match the one-shot request") } return Event{RequestIDs: slices.Clone(ids), SessionID: c.sessionID}, nil @@ -133,7 +172,16 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr var set *scaleset.RunnerScaleSet if err := d.effect(ctx, "observe-owned", nil, func(call context.Context) (Event, error) { var err error - set, err = d.API.GetScaleSet(call, setID) + if reader, ok := d.API.(drainScaleSetWireReader); ok { + wire := &baselineWireCapture{stage: "set-observe", setID: setID} + set, err = reader.drainGetScaleSet(call, setID, wire) + wireSet, status := wire.setFacts() + if err != nil || !wire.observed() || status != 200 || !wireSet.eligibleForDrain(d.Approval, setID) || !wireSet.matches(set) { + return Event{}, ErrRemote + } + } else { + set, err = d.API.GetScaleSet(call, setID) + } if err != nil || set == nil || set.ID != setID || set.Name != d.Approval.setName() || set.RunnerGroupID != d.Approval.RunnerGroupID || !set.RunnerSetting.DisableUpdate { return Event{}, ErrRemote } @@ -252,7 +300,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return err } - journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID, hook: hook} + journaled := &journaledDrainClient{d: d, inner: session, sessionID: sessionID, setID: setID, hook: hook} obs, runErr := runDrainListener(ctx, journaled, setID, hook) // The bounded observation sequence is phase-local: it must identify the // exact durable drain phase that was active when the listener ran. diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go new file mode 100644 index 0000000..c0318df --- /dev/null +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -0,0 +1,292 @@ +package livecanary + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "testing" + + "github.com/actions/scaleset" +) + +func TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner(t *testing.T) { + a := approval() + for _, tc := range []struct { + name string + mutate func(*http.Request) + }{ + { + name: "withdrawn capacity header", + mutate: func(req *http.Request) { + if req.Method == http.MethodGet && req.URL.Path == "/queue" && req.URL.Query().Get("lastMessageId") != "" { + req.Header.Set("X-ScaleSetMaxCapacity", "1") + } + }, + }, + { + name: "withdrawn cursor", + mutate: func(req *http.Request) { + if req.Method == http.MethodGet && req.URL.Path == "/queue" && req.URL.Query().Get("lastMessageId") != "" { + q := req.URL.Query() + q.Set("lastMessageId", "18") + req.URL.RawQuery = q.Encode() + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, _, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{mutateRequest: tc.mutate}) + observation, err := runDrainListener(context.Background(), session, 7, hook) + if !errors.Is(err, ErrQuarantine) || observation.Outcome == drainOutcomeObserved { + t.Fatalf("physical poll mutation was promoted: observation=%+v err=%v", observation, err) + } + if got := fixture.polls.Load(); got != 1 { + t.Fatalf("mutated withdrawn poll reached server: polls=%d, want first poll only", got) + } + }) + } +} + +func TestDrainCancellationStopsBeforeReleasingHeldResponse(t *testing.T) { + var order []string + cancelAndJoinDrain( + func() { order = append(order, "cancel") }, + func() { order = append(order, "release") }, + func() { order = append(order, "join") }, + ) + if got, want := strings.Join(order, ","), "cancel,release,join"; got != want { + t.Fatalf("cancellation order = %s, want %s", got, want) + } +} + +func TestPinnedSDKDrainRejectsWithdrawnPollBodyBeforeAbsent(t *testing.T) { + a := approval() + job := pinnedDrainJobBody(a) + body := fmt.Sprintf(`{"messageId":18,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":%s}`, mustJSONQuote(job)) + fixture, _, session, hook := newPinnedDrainSessionOptions(t, a, job, pinnedDrainOptions{withdrawnBody: body}) + observation, err := runDrainListener(context.Background(), session, 7, hook) + if !errors.Is(err, ErrQuarantine) || observation.Outcome == drainOutcomeObserved { + t.Fatalf("withdrawn 202 body was classified as absent: observation=%+v err=%v", observation, err) + } + if fixture.polls.Load() != 2 || fixture.acks.Load() != 1 || fixture.acquires.Load() != 1 { + t.Fatalf("withdrawn body effects = polls %d ack %d acquire %d, want one bounded poll plus old effects", fixture.polls.Load(), fixture.acks.Load(), fixture.acquires.Load()) + } +} + +func TestPinnedSDKDrainRequiresCompletePollStatsBeforeVerifyRun(t *testing.T) { + a := approval() + job := pinnedDrainJobBody(a) + first := fmt.Sprintf(`{"messageId":17,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalRegisteredRunners":1,"totalIdleRunners":1},"body":%s}`, mustJSONQuote(job)) + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, job, pinnedDrainOptions{firstPollBody: first}) + d := &Driver{Approval: a, Journal: &memoryJournal{}, API: base} + c := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), hook: hook} + hook.releaseResponse() + if _, err := c.GetMessage(context.Background(), 0, drainInitialCapacity); !errors.Is(err, ErrQuarantine) { + t.Fatalf("incomplete poll statistics = %v, want quarantine", err) + } + if fixture.verifyCalls.Load() != 0 { + t.Fatalf("incomplete poll statistics crossed VerifyRun: calls=%d", fixture.verifyCalls.Load()) + } + if fixture.acks.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("incomplete poll statistics reached effects: ack=%d acquire=%d", fixture.acks.Load(), fixture.acquires.Load()) + } +} + +func TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects(t *testing.T) { + a := approval() + for _, tc := range []struct { + name string + body string + good bool + }{ + {name: "success", body: pinnedDrainRunJSON(a), good: true}, + {name: "duplicate exact head", body: ambiguousRunJSON(a, "duplicate-head"), good: false}, + {name: "duplicate casefold head", body: ambiguousRunJSON(a, "casefold-head"), good: false}, + {name: "null", body: "null", good: false}, + {name: "missing head", body: ambiguousRunJSON(a, "missing-head"), good: false}, + {name: "contradictory head", body: ambiguousRunJSON(a, "wrong-head"), good: false}, + {name: "malformed", body: `{"id":5`, good: false}, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{verifyBody: tc.body}) + d := &Driver{Approval: a, Journal: &memoryJournal{}, API: base} + c := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), hook: hook} + hook.releaseResponse() + _, err := c.GetMessage(context.Background(), 0, drainInitialCapacity) + if tc.good { + if err != nil { + t.Fatalf("valid VerifyRun = %v", err) + } + return + } + if !errors.Is(err, ErrApproval) && !errors.Is(err, ErrQuarantine) { + t.Fatalf("ambiguous VerifyRun = %v, want fixed rejection", err) + } + if fixture.acks.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("ambiguous VerifyRun reached effects: ack=%d acquire=%d", fixture.acks.Load(), fixture.acquires.Load()) + } + }) + } +} + +func TestPinnedSDKDrainAcquisitionRequiresStrictWireResponse(t *testing.T) { + a := approval() + for _, tc := range []struct { + name string + body string + good bool + }{ + {name: "success", body: `{"count":1,"value":[41]}`, good: true}, + {name: "duplicate count", body: `{"count":0,"count":1,"value":[41]}`, good: false}, + {name: "casefold count", body: `{"Count":0,"count":1,"value":[41]}`, good: false}, + {name: "duplicate value", body: `{"count":1,"value":[99],"value":[41]}`, good: false}, + {name: "casefold value", body: `{"count":1,"Value":[99],"value":[41]}`, good: false}, + {name: "missing count", body: `{"value":[41]}`, good: false}, + {name: "count mismatch", body: `{"count":2,"value":[41]}`, good: false}, + {name: "null count", body: `{"count":null,"value":[41]}`, good: false}, + {name: "malformed", body: `{"count":1,"value":[41]`, good: false}, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{acquireBody: tc.body}) + journal := &memoryJournal{} + d := &Driver{Approval: a, Journal: journal, API: base} + c := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), hook: hook} + hook.releaseResponse() + message, err := c.GetMessage(context.Background(), 0, drainInitialCapacity) + if err != nil || message == nil { + t.Fatalf("setup poll = message %v err %v", message, err) + } + if err := c.DeleteMessage(context.Background(), message.MessageID); err != nil { + t.Fatalf("setup ACK = %v", err) + } + _, err = c.AcquireJobs(context.Background(), []int64{41}) + if tc.good { + if err != nil { + t.Fatalf("valid acquisition = %v", err) + } + return + } + if !errors.Is(err, ErrQuarantine) { + t.Fatalf("ambiguous acquisition = %v, want quarantine", err) + } + if fixture.acquires.Load() != 1 { + t.Fatalf("ambiguous acquisition request count = %d, want one remote attempt", fixture.acquires.Load()) + } + if state := replay(journal.Events()); !state.uncertain { + t.Fatal("ambiguous acquisition did not retain uncertainty") + } + for _, event := range journal.Events() { + if event.Kind == "result" && event.Operation == "acquire" { + t.Fatal("ambiguous acquisition recorded a successful result") + } + } + }) + } +} + +func TestPinnedSDKDrainSnapshotsRequireStrictWireFacts(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(string) string + }{ + {name: "duplicate id", mutate: func(body string) string { return strings.Replace(body, `"id":7`, `"id":7,"id":7`, 1) }}, + {name: "casefold id", mutate: func(body string) string { return strings.Replace(body, `"id":7`, `"ID":99,"id":7`, 1) }}, + {name: "duplicate statistics", mutate: func(body string) string { + return strings.Replace(body, `"totalIdleRunners":1`, `"totalIdleRunners":0,"totalIdleRunners":1`, 1) + }}, + {name: "casefold statistics", mutate: func(body string) string { + return strings.Replace(body, `"totalIdleRunners":1`, `"TotalIdleRunners":0,"totalIdleRunners":1`, 1) + }}, + {name: "missing statistics", mutate: func(body string) string { + return strings.Replace(body, `,"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}`, "", 1) + }}, + {name: "null statistics", mutate: func(body string) string { + return strings.Replace(body, `"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}`, `"statistics":null`, 1) + }}, + } { + t.Run(tc.name, func(t *testing.T) { + a := approval() + valid := pinnedDrainSnapshotJSON(a) + fixture, base, _, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{snapshotBodies: []string{tc.mutate(valid)}}) + api := fixtureSDK{base} + a.Phases = append(a.Phases, "drain") + j := &memoryJournal{events: []Event{ + {Kind: "phase", Operation: "create"}, + {Kind: "intent", Operation: "create"}, + {Kind: "result", Operation: "create", ID: 7}, + }} + d := Driver{Approval: a, Journal: j, API: api} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("ambiguous snapshot = %v, want quarantine", err) + } + if fixture.polls.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("ambiguous snapshot reached session effects: polls=%d acquire=%d", fixture.polls.Load(), fixture.acquires.Load()) + } + _ = hook + }) + } +} + +func mustJSONQuote(value string) string { + data, err := json.Marshal(value) + if err != nil { + panic(err) + } + return string(data) +} + +func pinnedDrainRunJSON(a Approval) string { + data, _ := json.Marshal(pinnedDrainRun(a)) + return string(data) +} + +func ambiguousRunJSON(a Approval, mode string) string { + if mode == "missing-head" { + run := pinnedDrainRun(a) + delete(run, "head_sha") + data, _ := json.Marshal(run) + return string(data) + } + body := pinnedDrainRunJSON(a) + switch mode { + case "duplicate-head": + return strings.Replace(body, `"head_sha":"`+a.WorkflowSHA+`"`, `"head_sha":"foreign","head_sha":"`+a.WorkflowSHA+`"`, 1) + case "casefold-head": + return strings.Replace(body, `"head_sha":"`+a.WorkflowSHA+`"`, `"HEAD_SHA":"foreign","head_sha":"`+a.WorkflowSHA+`"`, 1) + case "wrong-head": + return strings.Replace(body, a.WorkflowSHA, strings.Repeat("f", 40), 1) + default: + return body + } +} + +type pinnedDrainStatsWire struct { + Available int `json:"totalAvailableJobs"` + Acquired int `json:"totalAcquiredJobs"` + Assigned int `json:"totalAssignedJobs"` + Running int `json:"totalRunningJobs"` + Registered int `json:"totalRegisteredRunners"` + Busy int `json:"totalBusyRunners"` + Idle int `json:"totalIdleRunners"` +} + +type pinnedDrainSnapshotWire struct { + ID int `json:"id"` + Name string `json:"name"` + RunnerGroupID int `json:"runnerGroupId"` + Labels []scaleset.Label `json:"labels"` + RunnerSetting scaleset.RunnerSetting `json:"RunnerSetting"` + Statistics pinnedDrainStatsWire `json:"statistics"` +} + +func pinnedDrainSnapshotJSON(a Approval) string { + data, _ := json.Marshal(pinnedDrainSnapshotWire{ + ID: 7, Name: a.setName(), RunnerGroupID: a.RunnerGroupID, + Labels: []scaleset.Label{{Name: a.setName(), Type: "System"}}, RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, + Statistics: pinnedDrainStatsWire{Registered: 1, Idle: 1}, + }) + return string(data) +} diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index 2943a9d..fa29a51 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "net/http/httptrace" + "net/url" "strconv" "strings" "sync" @@ -669,6 +670,7 @@ func TestDrainPollHookPreservesStatisticsFieldPresence(t *testing.T) { if err != nil { t.Fatal(err) } + req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(drainInitialCapacity)) response, err := hook.RoundTrip(req) if err != nil { t.Fatal(err) @@ -768,7 +770,7 @@ func (t *drainPhysicalWriteTransport) RoundTrip(req *http.Request) (*http.Respon if t.polls == 1 { return &http.Response{ StatusCode: http.StatusOK, - Body: io.NopCloser(strings.NewReader(`{"statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`)), + Body: io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[]"}`)), }, nil } return &http.Response{ @@ -825,6 +827,7 @@ func TestDrainPollHookRejectsDuplicatePhysicalWrites(t *testing.T) { if err != nil { t.Fatal(err) } + req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(drainInitialCapacity)) response, err := hook.RoundTrip(req) if err != nil { t.Fatal(err) @@ -872,12 +875,17 @@ func TestDrainPollHookRequiresOneSuccessfulWritePerPoll(t *testing.T) { if err != nil { t.Fatal(err) } + request.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(drainInitialCapacity)) firstResponse, _ := hook.RoundTrip(request) if firstResponse != nil { hook.releaseResponse() _, _ = io.ReadAll(firstResponse.Body) _ = firstResponse.Body.Close() + query := request.URL.Query() + query.Set("lastMessageId", "7") + request.URL.RawQuery = query.Encode() } + request.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(drainWithdrawnCapacity)) secondResponse, _ := hook.RoundTrip(request) if secondResponse != nil { _, _ = io.ReadAll(secondResponse.Body) @@ -892,6 +900,9 @@ func TestDrainPollHookRequiresOneSuccessfulWritePerPoll(t *testing.T) { if behavior == drainWriteDuplicate || behavior == drainWriteRetry { want = 2 } + if index == 1 && first == drainWriteError { + want = 0 + } if got := hook.wroteCallbacks[index+1]; got != want { t.Fatalf("poll %d callbacks = %d, want %d", index+1, got, want) } @@ -1149,11 +1160,20 @@ func (s *drainSyntheticSession) Close(context.Context) error { } return nil } -func (s *drainSyntheticSession) GetMessage(ctx context.Context, _, capacity int) (*scaleset.RunnerScaleSetMessage, error) { +func (s *drainSyntheticSession) GetMessage(ctx context.Context, last, capacity int) (*scaleset.RunnerScaleSetMessage, error) { if s.client == nil { return nil, errors.New("synthetic transport unavailable") } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.initial.MessageQueueURL, nil) + target, err := url.Parse(s.initial.MessageQueueURL) + if err != nil { + return nil, err + } + if last > 0 { + query := target.Query() + query.Set("lastMessageId", strconv.Itoa(last)) + target.RawQuery = query.Encode() + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) if err != nil { return nil, err } diff --git a/experiments/g01-scaleset/livecanary/observer_http.go b/experiments/g01-scaleset/livecanary/observer_http.go index c5803c1..88d45d8 100644 --- a/experiments/g01-scaleset/livecanary/observer_http.go +++ b/experiments/g01-scaleset/livecanary/observer_http.go @@ -82,6 +82,10 @@ func (r observedRepository) policyValue() repository { return repository{ID: r.ID, Private: r.Private != nil && *r.Private, Fork: r.Fork != nil && *r.Fork} } func (a *SDKAPI) observeApprovedRun(ctx context.Context) (observationResponse, error) { + return a.observeApprovedRunFor(ctx, a.approval, a.approval.WorkflowRunID) +} + +func (a *SDKAPI) observeApprovedRunFor(ctx context.Context, approval Approval, id int64) (observationResponse, error) { var wire struct { ID int64 `json:"id"` HeadSHA string `json:"head_sha"` @@ -91,12 +95,12 @@ func (a *SDKAPI) observeApprovedRun(ctx context.Context) (observationResponse, e Repository observedRepository `json:"repository"` HeadRepository observedRepository `json:"head_repository"` } - out, err := a.observationGET(ctx, "/repos/"+a.approval.Organization+"/"+a.approval.Repository+"/actions/runs/"+strconv.FormatInt(a.approval.WorkflowRunID, 10), a.credentials.VerificationToken, "rest_run", &wire) + out, err := a.observationGET(ctx, "/repos/"+approval.Organization+"/"+approval.Repository+"/actions/runs/"+strconv.FormatInt(id, 10), a.credentials.VerificationToken, "rest_run", &wire) if err != nil || out.Outcome == observationNotFound { return out, err } run := workflowRun{ID: wire.ID, HeadSHA: wire.HeadSHA, Event: wire.Event, Path: wire.Path, RunAttempt: wire.RunAttempt, Repository: wire.Repository.policyValue(), HeadRepository: wire.HeadRepository.policyValue()} - if wire.Repository.Private == nil || wire.Repository.Fork == nil || wire.HeadRepository.Private == nil || wire.HeadRepository.Fork == nil || !matchesApprovedRun(a.approval, a.approval.WorkflowRunID, run) { + if wire.Repository.Private == nil || wire.Repository.Fork == nil || wire.HeadRepository.Private == nil || wire.HeadRepository.Fork == nil || !matchesApprovedRun(approval, id, run) { return out, ErrRemote } return out, nil diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index c17cfba..e04aad2 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -217,11 +217,8 @@ func (a *SDKAPI) VerifyRun(ctx context.Context, approval Approval, id int64) err if id <= 0 || id != approval.WorkflowRunID { return ErrApproval } - var run workflowRun - if a.get(ctx, "/repos/"+approval.Organization+"/"+approval.Repository+"/actions/runs/"+strconv.FormatInt(id, 10), a.credentials.VerificationToken, &run) != nil { - return ErrApproval - } - if !matchesApprovedRun(approval, id, run) { + out, err := a.observeApprovedRunFor(ctx, approval, id) + if err != nil || out.Outcome == observationNotFound { return ErrApproval } return nil @@ -285,6 +282,13 @@ func (a *SDKAPI) FindScaleSet(c context.Context, name string, group int) (*scale func (a *SDKAPI) GetScaleSet(c context.Context, id int) (*scaleset.RunnerScaleSet, error) { return a.client.GetRunnerScaleSetByID(c, id) } + +func (a *SDKAPI) drainGetScaleSet(c context.Context, id int, wire *baselineWireCapture) (*scaleset.RunnerScaleSet, error) { + if wire == nil { + return nil, ErrQuarantine + } + return a.GetScaleSet(wire.context(c), id) +} func (a *SDKAPI) CreateScaleSet(c context.Context, s *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) { return a.client.CreateRunnerScaleSet(c, s) } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index ca58343..32b16ac 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "errors" + "io" "log/slog" "net" "net/http" @@ -482,20 +483,44 @@ func pinnedDrainJobBody(a Approval) string { return string(data) } +type pinnedDrainOptions struct { + firstPollBody string + withdrawnBody string + acquireBody string + verifyBody string + snapshotBodies []string + mutateRequest func(*http.Request) +} + type pinnedDrainFixture struct { - server *httptest.Server - body string - polls atomic.Int32 - acks atomic.Int32 - acquires atomic.Int32 - mu sync.Mutex - cursors []string - capacity []string + server *httptest.Server + body string + firstPollBody string + withdrawnBody string + acquireBody string + verifyBody string + snapshotBodies []string + polls atomic.Int32 + acks atomic.Int32 + acquires atomic.Int32 + verifyCalls atomic.Int32 + snapshotReads atomic.Int32 + mu sync.Mutex + cursors []string + capacity []string } func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainFixture, *SDKAPI, Session, *drainPollHook) { + return newPinnedDrainSessionOptions(t, a, body, pinnedDrainOptions{}) +} + +func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options pinnedDrainOptions) (*pinnedDrainFixture, *SDKAPI, Session, *drainPollHook) { t.Helper() - fixture := &pinnedDrainFixture{body: body} + fixture := &pinnedDrainFixture{ + body: body, firstPollBody: options.firstPollBody, withdrawnBody: options.withdrawnBody, + acquireBody: options.acquireBody, verifyBody: options.verifyBody, + snapshotBodies: append([]string(nil), options.snapshotBodies...), + } fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.HasSuffix(r.URL.Path, "/actions/runners/registration-token") && r.Method == http.MethodPost: @@ -514,8 +539,17 @@ func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainF _ = json.NewEncoder(w).Encode(scaleset.RunnerScaleSetSession{ SessionID: uuid.MustParse("00000000-0000-4000-8000-000000000021"), OwnerName: a.setName(), MessageQueueURL: fixture.server.URL + "/queue", MessageQueueAccessToken: "fixture-queue", - Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + RunnerScaleSet: pinnedDrainScaleSet(a), + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, }) + case strings.HasSuffix(r.URL.Path, "/actions/runs/5") && r.Method == http.MethodGet: + fixture.verifyCalls.Add(1) + w.Header().Set("Content-Type", "application/json") + if fixture.verifyBody != "" { + _, _ = io.WriteString(w, fixture.verifyBody) + return + } + _ = json.NewEncoder(w).Encode(pinnedDrainRun(a)) case r.URL.Path == "/queue" && r.Method == http.MethodGet: fixture.polls.Add(1) fixture.mu.Lock() @@ -523,6 +557,11 @@ func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainF fixture.capacity = append(fixture.capacity, r.Header.Get(scaleset.HeaderScaleSetMaxCapacity)) fixture.mu.Unlock() if fixture.polls.Load() == 1 { + if fixture.firstPollBody != "" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, fixture.firstPollBody) + return + } w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]any{ "messageId": 17, "messageType": "RunnerScaleSetJobMessages", "body": fixture.body, @@ -532,6 +571,10 @@ func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainF } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusAccepted) + if fixture.withdrawnBody != "" { + _, _ = io.WriteString(w, fixture.withdrawnBody) + return + } _ = json.NewEncoder(w).Encode(map[string]any{"statistics": pinnedDrainStatistics(0, 0)}) case r.URL.Path == "/queue/17" && r.Method == http.MethodDelete: fixture.acks.Add(1) @@ -539,9 +582,25 @@ func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainF case strings.HasSuffix(r.URL.Path, "/acquirejobs") && r.Method == http.MethodPost: fixture.acquires.Add(1) w.Header().Set("Content-Type", "application/json") + if fixture.acquireBody != "" { + _, _ = io.WriteString(w, fixture.acquireBody) + return + } _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []int64{41}}) case strings.Contains(r.URL.Path, "/sessions/") && r.Method == http.MethodDelete: w.WriteHeader(http.StatusNoContent) + case strings.HasSuffix(r.URL.Path, "/runnerscalesets/7") && r.Method == http.MethodGet: + read := int(fixture.snapshotReads.Add(1)) + if read <= len(fixture.snapshotBodies) && fixture.snapshotBodies[read-1] != "" { + w.Header().Set("Content-Type", "application/json") + _, _ = io.WriteString(w, fixture.snapshotBodies[read-1]) + return + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(pinnedDrainSnapshot(a)) + case strings.HasSuffix(r.URL.Path, "/agents") && r.Method == http.MethodGet: + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []any{map[string]any{"id": 19, "name": a.workerName(), "runnerScaleSetId": 7}}}) default: http.NotFound(w, r) } @@ -568,13 +627,18 @@ func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainF return hook }) } + if options.mutateRequest != nil { + wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { + return sdkRequestMutationRoundTripper{inner: inner, mutate: options.mutateRequest} + }) + } retry.HTTPClient.Transport = withResponseBudget(transport, wrappers...) options := []scaleset.HTTPOption{scaleset.WithRetryableHTTPClint(retry), scaleset.WithLogger(slog.New(slog.DiscardHandler))} client, err := scaleset.NewClientWithPersonalAccessToken(scaleset.NewClientWithPersonalAccessTokenConfig{GitHubConfigURL: fixture.server.URL + "/fixture-org", PersonalAccessToken: "synthetic-installation"}, options...) if err != nil { return nil, err } - return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: fixture.server.URL + "/api/v3", approval: a, options: options}, nil + return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: fixture.server.URL + "/api/v3", approval: a, credentials: credentials(a), options: options}, nil } base, err := buildAPI(nil) if err != nil { @@ -589,6 +653,27 @@ func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainF return fixture, base, session, hook } +type sdkRequestMutationRoundTripper struct { + inner http.RoundTripper + mutate func(*http.Request) +} + +func (t sdkRequestMutationRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if t.mutate != nil { + t.mutate(req) + } + return t.inner.RoundTrip(req) +} + +func pinnedDrainRun(a Approval) map[string]any { + repository := map[string]any{"id": a.RepositoryID, "private": true, "fork": false} + return map[string]any{ + "id": a.WorkflowRunID, "head_sha": a.WorkflowSHA, "event": "workflow_dispatch", + "path": a.WorkflowPath, "run_attempt": 1, "repository": repository, + "head_repository": repository, + } +} + func newPinnedDrainClient(session Session, hook *drainPollHook) *drainClient { initial := session.Session() stats, _ := newDrainStatistics(initial.Statistics) @@ -605,6 +690,24 @@ func pinnedDrainStatistics(available, assigned int) map[string]int { } } +func pinnedDrainScaleSet(a Approval) *scaleset.RunnerScaleSet { + return &scaleset.RunnerScaleSet{ + ID: 7, Name: a.setName(), RunnerGroupID: a.RunnerGroupID, + Labels: []scaleset.Label{{Name: a.setName(), Type: "System"}}, + RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + } +} + +func pinnedDrainSnapshot(a Approval) map[string]any { + return map[string]any{ + "id": 7, "name": a.setName(), "runnerGroupId": a.RunnerGroupID, + "labels": []map[string]string{{"name": a.setName(), "type": "System"}}, + "RunnerSetting": map[string]any{"disableUpdate": true}, + "statistics": pinnedDrainStatistics(0, 0), + } +} + func (f *pinnedDrainFixture) cursorsSnapshot() []string { f.mu.Lock() defer f.mu.Unlock() From 5ffaa604a17f16c5b77aebf088c7fcec47e816bd Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 06:47:25 +0900 Subject: [PATCH 18/63] fix(g01): bind acquire capture to actions endpoint --- .../livecanary/baseline_listener.go | 2 +- .../g01-scaleset/livecanary/baseline_wire.go | 26 +++++++-------- .../g01-scaleset/livecanary/drain_driver.go | 10 +++++- .../livecanary/drain_p1_followup_test.go | 32 +++++++++++++++++++ .../g01-scaleset/livecanary/drain_test.go | 18 ++++++++++- experiments/g01-scaleset/livecanary/sdk.go | 11 +++++++ 6 files changed, 83 insertions(+), 16 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_listener.go b/experiments/g01-scaleset/livecanary/baseline_listener.go index a674dc2..a839780 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener.go @@ -163,7 +163,7 @@ func (b *baselineListener) finish(r baselineRecord, known bool) (controllerRecor return ref, nil } func (b *baselineListener) wire(stage string) *baselineWireCapture { - return &baselineWireCapture{stage: stage, setID: b.setID, queue: b.queue} + return &baselineWireCapture{stage: stage, setID: b.setID, queue: b.queue, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} } func (b *baselineListener) run(after func(context.Context, baselineAcquisition) error) error { if b == nil || b.ctx == nil || b.cancel == nil || !b.mu.TryLock() { diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 5219c37..479ab25 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -8,6 +8,7 @@ import ( "io" "net/http" "net/url" + "slices" "strconv" "strings" "sync" @@ -20,6 +21,7 @@ type baselineWireCapture struct { setID int sessionID string queue string // private, captured from the exact session; never journaled + allowedHosts []string cursor int count, status int invalid bool @@ -69,20 +71,10 @@ func (c *baselineWireCapture) target(r *http.Request) bool { if c.stage == "session-open" { suffix += "sessions" } else if c.stage == "acquire" { - // The synthetic drain transport uses the queue URL for its bounded - // acquisition request. The released SDK uses the Actions API endpoint - // below; accepting this exact queue form keeps the strict adapter useful - // for both without widening the target matcher. - if c.queue != "" { - u, e := url.Parse(c.queue) - if e == nil && u.Scheme != "" && u.Host != "" && u.User == nil && u.Fragment == "" { - u.Path = strings.TrimSuffix(u.Path, "/") + "/acquirejobs" - if r.Method == "POST" && r.URL.String() == u.String() { - return true - } - } + if len(c.allowedHosts) == 0 || !slices.Contains(c.allowedHosts, r.URL.Host) { + return false } - suffix += "acquirejobs" + suffix = "/_apis/runtime" + suffix + "acquirejobs" } else if c.stage == "jit" { suffix += "generatejitconfig" } else { @@ -92,6 +84,14 @@ func (c *baselineWireCapture) target(r *http.Request) bool { return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && len(q) == 1 && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" } +func baselineWireAllowedHosts(a Approval, apiHost string) []string { + hosts := slices.Clone(a.ActionsHosts) + if apiHost != "" && !slices.Contains(hosts, apiHost) { + hosts = append(hosts, apiHost) + } + return hosts +} + // Run after the ordinary response budget has wrapped the body. Only this // explicitly marked exact request is read ahead; the SDK receives identical // bytes. Nothing is captured from bootstrap, unrelated or unmarked traffic. diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 32f4bec..9455459 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -16,6 +16,10 @@ type drainScaleSetWireReader interface { drainGetScaleSet(context.Context, int, *baselineWireCapture) (*scaleset.RunnerScaleSet, error) } +type drainEndpointHostReader interface { + drainEndpointHost() string +} + type journaledDrainClient struct { d *Driver inner Session @@ -143,7 +147,11 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] c.hook.mu.Lock() queue := c.hook.target c.hook.mu.Unlock() - wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue} + apiHost := "" + if reader, ok := c.d.API.(drainEndpointHostReader); ok { + apiHost = reader.drainEndpointHost() + } + wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} call = wire.context(call) } got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index c0318df..9d6264c 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -12,6 +12,38 @@ import ( "github.com/actions/scaleset" ) +func TestBaselineAcquireTargetIsActionsOnly(t *testing.T) { + capture := &baselineWireCapture{ + stage: "acquire", + setID: 7, + queue: "https://queue.example/queue?access_token=fixture", + allowedHosts: []string{"api.example"}, + } + for _, tc := range []struct { + name string + method string + target string + want bool + }{ + {name: "actions endpoint", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", want: true}, + {name: "queue endpoint", method: http.MethodPost, target: "https://queue.example/queue/acquirejobs?access_token=fixture", want: false}, + {name: "wrong host", method: http.MethodPost, target: "https://other.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", want: false}, + {name: "wrong set path", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/8/acquirejobs?api-version=6.0-preview", want: false}, + {name: "wrong endpoint path", method: http.MethodPost, target: "https://api.example/runnerscalesets/7/acquirejobs?api-version=6.0-preview", want: false}, + {name: "wrong method", method: http.MethodGet, target: "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequest(tc.method, tc.target, nil) + if err != nil { + t.Fatal(err) + } + if got := capture.target(req); got != tc.want { + t.Fatalf("acquisition target match = %v, want %v for %s", got, tc.want, tc.target) + } + }) + } +} + func TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner(t *testing.T) { a := approval() for _, tc := range []struct { diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index fa29a51..d1ceca3 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -25,6 +25,7 @@ func TestDrainListenerWithdrawsWhilePollResponseIsHeld(t *testing.T) { var mu sync.Mutex var capacities []string var order []string + var acquireRequests []string polls := 0 message := map[string]any{ "messageId": 7, @@ -67,6 +68,7 @@ func TestDrainListenerWithdrawsWhilePollResponseIsHeld(t *testing.T) { } if r.Method == http.MethodPost { order = append(order, "acquire") + acquireRequests = append(acquireRequests, r.Method+" "+r.URL.RequestURI()) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) _, _ = io.WriteString(w, `{"count":1,"value":[11]}`) @@ -96,6 +98,7 @@ func TestDrainListenerWithdrawsWhilePollResponseIsHeld(t *testing.T) { mu.Lock() gotCapacities := append([]string(nil), capacities...) gotOrder := append([]string(nil), order...) + gotAcquireRequests := append([]string(nil), acquireRequests...) mu.Unlock() if len(gotCapacities) != 2 || gotCapacities[0] != "1" || gotCapacities[1] != "0" { t.Fatalf("poll capacities = %v, want [1 0]", gotCapacities) @@ -103,6 +106,9 @@ func TestDrainListenerWithdrawsWhilePollResponseIsHeld(t *testing.T) { if len(gotOrder) != 2 || gotOrder[0] != "ack" || gotOrder[1] != "acquire" { t.Fatalf("side-effect order = %v, want ACK then acquire", gotOrder) } + if len(gotAcquireRequests) != 1 || gotAcquireRequests[0] != "POST /_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview" { + t.Fatalf("acquisition request = %v, want pinned Actions endpoint", gotAcquireRequests) + } if observation.Boundary != drainBoundaryRequestWritten || !observation.ResponseHeld || observation.ServerReceipt != drainServerReceiptUnproven { t.Fatalf("boundary = %+v, want request-written/held/server-unproven", observation) } @@ -1211,7 +1217,17 @@ func (s *drainSyntheticSession) AcquireJobs(ctx context.Context, ids []int64) ([ if len(ids) != 1 { return nil, errors.New("unexpected request count") } - if err := appendSyntheticMethod(ctx, s.client, http.MethodPost, s.initial.MessageQueueURL+"/acquirejobs", s.order); err != nil { + target, err := url.Parse(s.initial.MessageQueueURL) + if err != nil { + return nil, err + } + setID := 7 + if s.initial.RunnerScaleSet != nil && s.initial.RunnerScaleSet.ID > 0 { + setID = s.initial.RunnerScaleSet.ID + } + target.Path = "/_apis/runtime/runnerscalesets/" + strconv.Itoa(setID) + "/acquirejobs" + target.RawQuery = url.Values{"api-version": {"6.0-preview"}}.Encode() + if err := appendSyntheticMethod(ctx, s.client, http.MethodPost, target.String(), s.order); err != nil { return nil, err } return append([]int64(nil), ids...), nil diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index e04aad2..c2813a8 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -283,6 +283,17 @@ func (a *SDKAPI) GetScaleSet(c context.Context, id int) (*scaleset.RunnerScaleSe return a.client.GetRunnerScaleSetByID(c, id) } +func (a *SDKAPI) drainEndpointHost() string { + if a == nil { + return "" + } + u, err := url.Parse(a.baseURL) + if err != nil || u.Scheme == "" || u.Host == "" || u.User != nil || u.Fragment != "" { + return "" + } + return u.Host +} + func (a *SDKAPI) drainGetScaleSet(c context.Context, id int, wire *baselineWireCapture) (*scaleset.RunnerScaleSet, error) { if wire == nil { return nil, ErrQuarantine From 9904048a8250f2445f5fee4c2141b3f9f1642e3a Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 06:49:03 +0900 Subject: [PATCH 19/63] docs(g01): record acquire target correction --- docs/evidence/g01-idle-drain.md | 52 +++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 0fec7c3..e30a137 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -159,6 +159,58 @@ than being accepted as a complete statistics sample. The historical reds are not relabeled as pre-implementation tests for this follow-up; the actual red run above is the new TDD regression, followed by the listed green runs. +### Acquire target correction (exact head `9b1f0c214022740d6276b6f9dd417f4f892d0bd5`) + +The follow-up review identified a production-boundary leak in the bounded +acquisition capture: `baselineWireCapture.target` accepted a synthetic +queue-URL `/acquirejobs` request, and its suffix-only Actions matcher accepted +the same endpoint shape on an arbitrary host. The pinned SDK source confirms +the real request is `POST /_apis/runtime/runnerscalesets/{setID}/acquirejobs` +with exactly `api-version=6.0-preview`; the strict `count`/`value` decoder and +the prior [acquisition P1 evidence](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911056) +remain unchanged. + +The meaningful red was run before the correction from the exact head above: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestBaselineAcquireTargetIsActionsOnly$' -count=1 -v +``` + +It failed because `queue_endpoint` and `wrong_host` both returned `true` +instead of `false`. The minimal source correction in +`5ffaa604a17f16c5b77aebf088c7fcec47e816bd` removes the queue compatibility +branch, requires the Actions runtime path and an approved/API host, and moves +the synthetic fixture to the pinned endpoint; queue query material is not +copied into that test-only request. The complete poll-statistics preflight +before `VerifyRun`, strict bounded acquisition decoding, and all six prior P1 +fixes remain in place. + +The focused green and safety checks were: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineAcquireTargetIsActionsOnly|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestPinnedSDKDrainAcquisitionRequiresStrictWireResponse)$' -count=1 -v -timeout=180s +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineAcquireTargetIsActionsOnly|TestPinnedSDKDrain.*|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDrainCancellationStopsBeforeReleasingHeldResponse)$' -count=1 -v -timeout=180s +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestBaselineAcquireTargetIsActionsOnly|TestPinnedSDKDrain.*|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestDrainCancellationStopsBeforeReleasingHeldResponse)$' -count=1 -timeout=180s +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +cd ../.. +test -z "$(gofmt -l experiments/g01-scaleset/livecanary/baseline_wire.go experiments/g01-scaleset/livecanary/baseline_listener.go experiments/g01-scaleset/livecanary/drain_driver.go experiments/g01-scaleset/livecanary/sdk.go experiments/g01-scaleset/livecanary/drain_test.go experiments/g01-scaleset/livecanary/drain_p1_followup_test.go)" +git diff --check +``` + +All focused normal/race tests passed, vet was silent and successful, and the +format/diff checks were clean. The queue endpoint, wrong host, wrong set/path, +and wrong method cases remain rejected while the actual pinned SDK endpoint and +the synthetic fixture endpoint are accepted; these are offline checks only. +No live operation, credential, runner, Docker, Keychain, launchd, workflow or +personal-path mutation occurred, and historical snapshots remain unchanged. +Rollback is recoverable with normal `git revert --no-edit +5ffaa604a17f16c5b77aebf088c7fcec47e816bd`, which returns the source to the +published `9b1f0c214022740d6276b6f9dd417f4f892d0bd5` correction head; the +documentation-only commit that records this evidence can be reverted +separately without rewriting history. + ## Independent correction matrix | Finding | Correction and evidence | From 19f53d4e7af9578c7719c7897aa032124df6df6b Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 07:40:31 +0900 Subject: [PATCH 20/63] fix(g01): close drain SDK evidence boundaries --- .../livecanary/baseline_message.go | 69 ++++++++- .../g01-scaleset/livecanary/baseline_wire.go | 13 ++ experiments/g01-scaleset/livecanary/drain.go | 12 +- .../g01-scaleset/livecanary/drain_driver.go | 30 +++- experiments/g01-scaleset/livecanary/sdk.go | 40 +++++- .../livecanary/sdk_integration_test.go | 135 +++++++++++++++++- 6 files changed, 291 insertions(+), 8 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_message.go b/experiments/g01-scaleset/livecanary/baseline_message.go index db07b71..41532ff 100644 --- a/experiments/g01-scaleset/livecanary/baseline_message.go +++ b/experiments/g01-scaleset/livecanary/baseline_message.go @@ -11,8 +11,9 @@ import ( "github.com/actions/scaleset" ) -// Pointer facts retain missing/null separately from explicit zero/false. No -// queue/acquisition URL, token, arbitrary display text or raw response is kept. +// Pointer facts retain missing/null separately from explicit zero/false. +// Persistent facts omit queue/acquisition URLs, tokens, arbitrary display text +// and raw responses. type baselineStatistics struct { Available *int `json:"total_available_jobs"` Acquired *int `json:"total_acquired_jobs"` @@ -55,7 +56,56 @@ type baselineSessionFacts struct { SetID int `json:"set_id"` SetName string `json:"set_name"` GroupID int `json:"group_id"` + queueURL string `json:"-"` // ephemeral drain validation only; never serialized } + +// baselineRunnerFacts retains only the bounded fields needed to compare the +// pinned SDK's runner lookup result with the exact wire response. The list +// count is kept separately because the SDK collapses count==0 to nil and +// otherwise returns only the first value. +type baselineRunnerFacts struct { + Count int + ID int + Name string + ScaleSetID int +} + +func decodeBaselineRunner(data []byte) (*baselineRunnerFacts, error) { + var w struct { + Count *int `json:"count"` + Value *[]struct { + ID *int `json:"id"` + Name *string `json:"name"` + ScaleSetID *int `json:"runnerScaleSetId"` + } `json:"value"` + } + if DecodeStrict(data, &w) != nil || w.Count == nil || w.Value == nil || *w.Count < 0 || *w.Count > 1 || *w.Count != len(*w.Value) { + return nil, ErrRemote + } + facts := &baselineRunnerFacts{Count: *w.Count} + if facts.Count == 0 { + return facts, nil + } + candidate := (*w.Value)[0] + if candidate.ID == nil || *candidate.ID <= 0 || candidate.Name == nil || !baselineText(*candidate.Name, 256) || candidate.ScaleSetID == nil || *candidate.ScaleSetID <= 0 { + return nil, ErrRemote + } + facts.ID = *candidate.ID + facts.Name = *candidate.Name + facts.ScaleSetID = *candidate.ScaleSetID + return facts, nil +} + +func (r *baselineRunnerFacts) matches(v *scaleset.RunnerReference) bool { + if r == nil { + return false + } + if r.Count == 0 { + return v == nil + } + return r.Count == 1 && v != nil && r.ID == v.ID && r.Name == v.Name && r.ScaleSetID == v.RunnerScaleSetID +} + type baselineAccepted struct { Count *int `json:"count"` IDs []int64 `json:"ids"` @@ -193,6 +243,19 @@ func (s *baselineStatistics) matches(sdk *scaleset.RunnerScaleSetStatistic) bool } return true } + +func (s *baselineStatistics) completeDrain() bool { + if s == nil { + return false + } + values := []*int{s.Available, s.Acquired, s.Assigned, s.Running, s.Registered, s.Busy, s.Idle} + for _, value := range values { + if value == nil || *value < 0 { + return false + } + } + return validKnownDrainStatistics(drainStatistics{Available: *s.Available, Acquired: *s.Acquired, Assigned: *s.Assigned, Running: *s.Running, Registered: *s.Registered, Busy: *s.Busy, Idle: *s.Idle}) +} func decodeBaselineItem(data []byte, index int) (baselineItem, error) { var w struct { Kind string `json:"messageType"` @@ -300,7 +363,7 @@ func decodeBaselineSession(data []byte) (*baselineSessionFacts, error) { if err != nil { return nil, err } - x := &baselineSessionFacts{SessionID: w.ID, Owner: w.Owner, Statistics: s} + x := &baselineSessionFacts{SessionID: w.ID, Owner: w.Owner, Statistics: s, queueURL: w.URL} if len(w.Set) > 0 && string(w.Set) != "null" { // Set contains SDK-defined fields not used here; ambiguity is rejected, // but unrelated forward-compatible set metadata is not copied to history. diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 479ab25..fae57ce 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -19,6 +19,7 @@ type baselineWireCapture struct { mu sync.Mutex stage string setID int + runnerName string sessionID string queue string // private, captured from the exact session; never journaled allowedHosts []string @@ -29,6 +30,7 @@ type baselineWireCapture struct { batch *baselineBatch accepted *baselineAccepted set *baselineSetFacts + runner *baselineRunnerFacts jit *scaleset.RunnerScaleSetJitRunnerConfig } @@ -70,6 +72,9 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } if c.stage == "session-open" { suffix += "sessions" + } else if c.stage == "runner-observe" { + q := r.URL.Query() + return r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/_apis/distributedtask/pools/0/agents") && len(q) == 2 && len(q["agentName"]) == 1 && q.Get("agentName") == c.runnerName && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" } else if c.stage == "acquire" { if len(c.allowedHosts) == 0 || !slices.Contains(c.allowedHosts, r.URL.Host) { return false @@ -124,6 +129,8 @@ func guardBaselineResponse(req *http.Request, response *http.Response) (*http.Re c.set, err = decodeBaselineSet(data) case "session-open": c.session, err = decodeBaselineSession(data) + case "runner-observe": + c.runner, err = decodeBaselineRunner(data) case "poll": c.batch, err = decodeBaselineBatch(data) case "jit": @@ -181,6 +188,12 @@ func (c *baselineWireCapture) setFacts() (*baselineSetFacts, int) { return c.set, c.status } +func (c *baselineWireCapture) runnerFacts() (*baselineRunnerFacts, int) { + c.mu.Lock() + defer c.mu.Unlock() + return c.runner, c.status +} + func validJITSecret(secret string) bool { if len(secret) < 16 || len(secret) > 1<<20 { return false diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 7a239a4..b780d10 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -619,6 +619,7 @@ type drainObservedBody struct { over bool mu sync.Mutex finished bool + readFailed bool onComplete func(drainStatistics, bool, *baselineBatch, bool, bool, bool) } @@ -647,6 +648,7 @@ func (b *drainObservedBody) finish() { b.finished = true data := append([]byte(nil), b.data...) over := b.over + readFailed := b.readFailed b.data = nil b.mu.Unlock() @@ -662,7 +664,7 @@ func (b *drainObservedBody) finish() { present = false absent = false } - if over { + if over || readFailed { known = false stats = drainStatistics{} batch = nil @@ -681,6 +683,11 @@ func (b *drainObservedBody) Read(p []byte) (int, error) { b.capture(p[:n]) } if err != nil { + if !errors.Is(err, io.EOF) { + b.mu.Lock() + b.readFailed = true + b.mu.Unlock() + } b.finish() } return n, err @@ -706,6 +713,9 @@ func (b *drainObservedBody) Close() error { } if err != nil { b.mu.Lock() + if !errors.Is(err, io.EOF) { + b.readFailed = true + } b.over = true b.mu.Unlock() } diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 9455459..8f34c17 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -3,6 +3,7 @@ package livecanary import ( "context" "errors" + "net/http" "slices" "github.com/actions/scaleset" @@ -16,6 +17,10 @@ type drainScaleSetWireReader interface { drainGetScaleSet(context.Context, int, *baselineWireCapture) (*scaleset.RunnerScaleSet, error) } +type drainRunnerWireReader interface { + drainFindRunner(context.Context, string, *baselineWireCapture) (*scaleset.RunnerReference, error) +} + type drainEndpointHostReader interface { drainEndpointHost() string } @@ -120,9 +125,23 @@ func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error return err } return c.d.effect(callCtx, "ack", nil, func(call context.Context) (Event, error) { + var wire *baselineWireCapture + if c.hook != nil { + c.hook.mu.Lock() + queue := c.hook.target + c.hook.mu.Unlock() + wire = &baselineWireCapture{stage: "ack", queue: queue, cursor: id} + call = wire.context(call) + } if err := c.inner.DeleteMessage(call, id); err != nil { return Event{}, err } + if wire != nil { + _, _, _, status := wire.facts() + if !wire.observed() || status != http.StatusNoContent { + return Event{}, errors.New("ack response did not match the one-shot request") + } + } return Event{ID: id, SessionID: c.sessionID}, nil }) } @@ -213,7 +232,16 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr var runner *scaleset.RunnerReference if err := d.effect(ctx, "observe-runner", nil, func(call context.Context) (Event, error) { var err error - runner, err = d.API.FindRunner(call, d.Approval.workerName()) + if reader, ok := d.API.(drainRunnerWireReader); ok { + wire := &baselineWireCapture{stage: "runner-observe", runnerName: d.Approval.workerName()} + runner, err = reader.drainFindRunner(call, d.Approval.workerName(), wire) + wireRunner, status := wire.runnerFacts() + if err != nil || !wire.observed() || status != http.StatusOK || !wireRunner.matches(runner) { + return Event{}, ErrQuarantine + } + } else { + runner, err = d.API.FindRunner(call, d.Approval.workerName()) + } if err != nil { return Event{}, err } diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index c2813a8..1bca7e2 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -300,6 +300,34 @@ func (a *SDKAPI) drainGetScaleSet(c context.Context, id int, wire *baselineWireC } return a.GetScaleSet(wire.context(c), id) } + +func (a *SDKAPI) drainFindRunner(c context.Context, name string, wire *baselineWireCapture) (*scaleset.RunnerReference, error) { + if a == nil || a.client == nil || wire == nil || name == "" { + return nil, ErrQuarantine + } + wire.runnerName = name + return a.client.GetRunnerByName(wire.context(c), name) +} + +func validDrainQueueURL(value string) bool { + if value == "" || !baselineText(value, 4096) { + return false + } + u, err := url.Parse(value) + return err == nil && u.Scheme != "" && u.Host != "" && u.User == nil && u.Fragment == "" && u.EscapedPath() == u.Path +} + +func validDrainSessionWire(a Approval, id int, owner string, wire *baselineSessionFacts, session scaleset.RunnerScaleSetSession) bool { + if wire == nil || session.SessionID == [16]byte{} || wire.SessionID != session.SessionID.String() || wire.Owner != owner || session.OwnerName != owner || !validDrainQueueURL(wire.queueURL) || wire.queueURL != session.MessageQueueURL || session.MessageQueueAccessToken == "" || !wire.Statistics.completeDrain() || !wire.NestedSet || wire.SetID != id || wire.SetName != owner || wire.GroupID != a.RunnerGroupID || !wire.NestedStatistics.completeDrain() || !wire.Statistics.matches(session.Statistics) { + return false + } + set := session.RunnerScaleSet + if set == nil || set.ID != id || set.Name != owner || set.RunnerGroupID != a.RunnerGroupID || !set.RunnerSetting.DisableUpdate || !slices.ContainsFunc(set.Labels, func(label scaleset.Label) bool { return label.Name == owner }) || set.Statistics == nil || !wire.NestedStatistics.matches(set.Statistics) { + return false + } + return true +} + func (a *SDKAPI) CreateScaleSet(c context.Context, s *scaleset.RunnerScaleSet) (*scaleset.RunnerScaleSet, error) { return a.client.CreateRunnerScaleSet(c, s) } @@ -324,12 +352,20 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook if err != nil { return nil, err } - session, err := configured.client.MessageSessionClient(c, id, owner, configured.options...) + if configured == nil || configured.client == nil { + return nil, ErrRemote + } + wire := &baselineWireCapture{stage: "session-open", setID: id} + session, err := configured.client.MessageSessionClient(wire.context(c), id, owner, configured.options...) if err != nil { return nil, ErrRemote } + sessionFacts, _, _, status := wire.facts() + if session == nil || !wire.observed() || status != http.StatusOK || !validDrainSessionWire(configured.approval, id, owner, sessionFacts, session.Session()) { + return nil, ErrQuarantine + } hook.mu.Lock() - hook.target = session.Session().MessageQueueURL + hook.target = sessionFacts.queueURL hook.mu.Unlock() return session, nil } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 32b16ac..3a422b2 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -472,6 +472,70 @@ func TestPinnedSDKDrainMatchesWireBeforeVerifyRun(t *testing.T) { } } +func TestPinnedSDKDrainRejectsNonEOFPollReadError(t *testing.T) { + a := approval() + fixture, _, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{pollReadError: io.ErrUnexpectedEOF}) + hook.releaseResponse() + if _, err := session.GetMessage(context.Background(), 0, drainInitialCapacity); err == nil { + t.Fatal("non-EOF poll read error was accepted by the pinned SDK") + } + if _, known := hook.pollStatistics(1); known { + t.Fatalf("non-EOF poll read error became known wire facts: fixture polls=%d", fixture.polls.Load()) + } + if fixture.acks.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("non-EOF poll read error reached effects: ack=%d acquire=%d", fixture.acks.Load(), fixture.acquires.Load()) + } +} + +func TestPinnedSDKDrainBindsACKToPhysicalDelete(t *testing.T) { + a := approval() + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + acceptAnyACK: true, + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodDelete && strings.HasPrefix(req.URL.Path, "/queue/") { + req.URL.Path = "/queue/999" + req.URL.RawPath = "" + } + }, + }) + hook.releaseResponse() + j := &memoryJournal{} + api := &countingFixtureSDK{fixtureSDK: fixtureSDK{base}} + d := &Driver{Approval: a, Journal: j, API: api} + client := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), setID: 7, hook: hook} + message, err := client.GetMessage(context.Background(), 0, drainInitialCapacity) + if err != nil || message == nil { + t.Fatalf("pinned SDK setup poll = message %v err %v", message, err) + } + if err := client.DeleteMessage(context.Background(), message.MessageID); !errors.Is(err, ErrQuarantine) { + t.Fatalf("wrong physical ACK = %v, want quarantine", err) + } + if fixture.acks.Load() != 1 { + t.Fatalf("wrong physical ACK did not reach successful fixture endpoint: ack=%d", fixture.acks.Load()) + } +} + +func TestPinnedSDKDrainRejectsAmbiguousRunnerSnapshot(t *testing.T) { + a := approval() + runnerBody := `{"count":1,"value":[{"id":19,"name":"` + a.workerName() + `","runnerScaleSetId":7,"RunnerScaleSetId":7}]}` + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{runnerBody: runnerBody}) + j := &memoryJournal{} + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + if _, err := d.drainSnapshot(context.Background(), 7, "before"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("ambiguous runner snapshot = %v, want quarantine (fixture polls=%d)", err, fixture.polls.Load()) + } +} + +func TestPinnedSDKDrainRejectsAmbiguousSessionResponse(t *testing.T) { + a := approval() + name := a.setName() + sessionBody := `{"sessionId":"00000000-0000-4000-8000-000000000021","SessionID":"00000000-0000-4000-8000-000000000021","ownerName":"` + name + `","runnerScaleSet":{"id":7,"name":"` + name + `","runnerGroupId":3,"labels":[{"name":"` + name + `","type":"System"}],"RunnerSetting":{"disableUpdate":true},"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}},"messageQueueUrl":"QUEUE_URL","MessageQueueURL":"QUEUE_URL","messageQueueAccessToken":"fixture-queue","statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}` + fixture, _, session, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{sessionBody: sessionBody, allowOpenError: true}) + if fixture.openErr == nil || session != nil { + t.Fatalf("ambiguous session response accepted: err=%v session=%v", fixture.openErr, session) + } +} + func pinnedDrainJobBody(a Approval) string { data, _ := json.Marshal([]map[string]any{{ "messageType": "JobAvailable", @@ -488,6 +552,11 @@ type pinnedDrainOptions struct { withdrawnBody string acquireBody string verifyBody string + sessionBody string + runnerBody string + allowOpenError bool + acceptAnyACK bool + pollReadError error snapshotBodies []string mutateRequest func(*http.Request) } @@ -499,6 +568,8 @@ type pinnedDrainFixture struct { withdrawnBody string acquireBody string verifyBody string + runnerBody string + openErr error snapshotBodies []string polls atomic.Int32 acks atomic.Int32 @@ -519,6 +590,7 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options fixture := &pinnedDrainFixture{ body: body, firstPollBody: options.firstPollBody, withdrawnBody: options.withdrawnBody, acquireBody: options.acquireBody, verifyBody: options.verifyBody, + runnerBody: options.runnerBody, snapshotBodies: append([]string(nil), options.snapshotBodies...), } fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -536,6 +608,10 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options }) case strings.HasSuffix(r.URL.Path, "/sessions") && r.Method == http.MethodPost: w.Header().Set("Content-Type", "application/json") + if options.sessionBody != "" { + _, _ = io.WriteString(w, strings.ReplaceAll(options.sessionBody, "QUEUE_URL", fixture.server.URL+"/queue")) + return + } _ = json.NewEncoder(w).Encode(scaleset.RunnerScaleSetSession{ SessionID: uuid.MustParse("00000000-0000-4000-8000-000000000021"), OwnerName: a.setName(), MessageQueueURL: fixture.server.URL + "/queue", MessageQueueAccessToken: "fixture-queue", @@ -576,6 +652,9 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options return } _ = json.NewEncoder(w).Encode(map[string]any{"statistics": pinnedDrainStatistics(0, 0)}) + case options.acceptAnyACK && strings.HasPrefix(r.URL.Path, "/queue/") && r.Method == http.MethodDelete: + fixture.acks.Add(1) + w.WriteHeader(http.StatusNoContent) case r.URL.Path == "/queue/17" && r.Method == http.MethodDelete: fixture.acks.Add(1) w.WriteHeader(http.StatusNoContent) @@ -600,6 +679,10 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options _ = json.NewEncoder(w).Encode(pinnedDrainSnapshot(a)) case strings.HasSuffix(r.URL.Path, "/agents") && r.Method == http.MethodGet: w.Header().Set("Content-Type", "application/json") + if fixture.runnerBody != "" { + _, _ = io.WriteString(w, fixture.runnerBody) + return + } _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []any{map[string]any{"id": 19, "name": a.workerName(), "runnerScaleSetId": 7}}}) default: http.NotFound(w, r) @@ -622,6 +705,11 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options } wrappers := []func(http.RoundTripper) http.RoundTripper(nil) if hook != nil { + if options.pollReadError != nil { + wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { + return sdkResponseBodyFaultRoundTripper{inner: inner, path: "/queue", err: options.pollReadError} + }) + } wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { hook.inner = inner return hook @@ -648,11 +736,56 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options hook := newDrainPollHook("") session, err := base.OpenDrainSession(context.Background(), 7, a.setName(), hook) if err != nil { - t.Fatal(err) + fixture.openErr = err + if !options.allowOpenError { + t.Fatal(err) + } + return fixture, base, nil, hook } return fixture, base, session, hook } +type sdkResponseBodyFaultRoundTripper struct { + inner http.RoundTripper + path string + err error +} + +func (t sdkResponseBodyFaultRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + response, err := t.inner.RoundTrip(req) + if err != nil || response == nil || req.URL == nil || req.URL.Path != t.path || response.Body == nil { + return response, err + } + response.Body = &sdkResponseBodyFault{source: response.Body, err: t.err} + return response, nil +} + +type sdkResponseBodyFault struct { + source io.ReadCloser + err error + injected bool + pending bool +} + +func (b *sdkResponseBodyFault) Read(p []byte) (int, error) { + if b.pending { + b.pending = false + return 0, b.err + } + n, err := b.source.Read(p) + if err == io.EOF && !b.injected { + b.injected = true + if n > 0 { + b.pending = true + return n, nil + } + return 0, b.err + } + return n, err +} + +func (b *sdkResponseBodyFault) Close() error { return b.source.Close() } + type sdkRequestMutationRoundTripper struct { inner http.RoundTripper mutate func(*http.Request) From 88d37abea8ba4d6b793c849b38bcf797f2dbb503 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 07:42:05 +0900 Subject: [PATCH 21/63] docs(g01): record SDK drain boundary evidence --- docs/evidence/g01-idle-drain.md | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index e30a137..9d9aa10 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -697,6 +697,97 @@ reset, erase, replay or run live cleanup. Historical evidence section contains no personal machine paths or raw secret-bearing payloads. +### Exact-head follow-up: SDK drain response and transport boundary closure + +The four latest exact-head Codex P1 findings were read from the PR review +record with `gh` against the immutable baseline +`9904048a8250f2445f5fee4c2141b3f9f1642e3a` (the local review wrapper was not +present, so the equivalent `gh api` detail/all reads were used): + +```text +gh api --paginate repos/1XP-AI/gh-runnerd/pulls/72/comments --jq '.[] | [.id,.path,.line,.html_url] | @tsv' +gh api repos/1XP-AI/gh-runnerd/pulls/comments/3973406578 --jq '{html_url,path,line,body}' +gh api repos/1XP-AI/gh-runnerd/pulls/comments/3973406571 --jq '{html_url,path,line,body}' +gh api repos/1XP-AI/gh-runnerd/pulls/comments/3973406564 --jq '{html_url,path,line,body}' +gh api repos/1XP-AI/gh-runnerd/pulls/comments/3973406553 --jq '{html_url,path,line,body}' +gh api --paginate repos/1XP-AI/gh-runnerd/issues/72/comments --jq '.[] | [.id,.html_url] | @tsv' +``` + +The findings are [non-EOF poll read error](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973406578), [ACK physical DELETE binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973406571), [runner snapshot decoding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973406564), and [session-open decoding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973406553). The historical TDD exception remains only the original [Issue #71 authorization](https://github.com/1XP-AI/gh-runnerd/issues/71#issuecomment-5603758575); it does not waive this follow-up or any future correction. + +Meaningful red regressions were added and run before implementation against +the real pinned `github.com/actions/scaleset v0.4.0` loopback fixture: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainBindsACKToPhysicalDelete|TestPinnedSDKDrainRejectsAmbiguousRunnerSnapshot|TestPinnedSDKDrainRejectsAmbiguousSessionResponse' -count=1 -v +``` + +The command exited 1 as intended. The old code reported known poll facts after +a complete JSON body followed by a non-EOF read error; accepted a mutated ACK +DELETE; accepted duplicate runner-set identity; and accepted duplicate session +identity/queue fields. The red assertions exercised the pinned SDK and loopback +wire path, used effect counters where applicable, and did not disclose private +response values. + +The minimal source/test correction is commit +`19f53d4e7af9578c7719c7897aa032124df6df6b`. `drainObservedBody` now records a +non-EOF read failure and clears all derived facts, so a lossy SDK success cannot +promote the response to known. The existing bounded `baselineWireCapture` +adapter is reused for session-open and ACK; session-open validates canonical +session identity, owner, nested set, complete statistics, and exact ephemeral +queue URL before assigning the poll hook, while ACK requires one exact physical +`DELETE` for the captured queue and message ID with status 204. A runner +equivalent uses the same strict duplicate-key decoder and compares the bounded +count/value identity to the pinned SDK result; an ambiguous runner or session +response is rejected before downstream effects, and a failed session-open +leaves the created remote session unclosed for quarantine inspection. + +The complete evidence-bearing drain inventory is now: + +| Boundary | Bounded evidence and gate | +| --- | --- | +| Before/after scale-set snapshot | `set-observe` strict bounded identity, labels, update fence, statistics, status 200, and SDK comparison. | +| Before/after runner lookup | `runner-observe` strict count/value (`nil` only for an exact empty list) and exact ID/name/scale-set comparison. | +| Session creation | `session-open` strict identity, nested set/statistics and queue URL comparison; queue URL and token are ephemeral and never journaled. | +| Old and withdrawn polls | Exact queue target, cursor, capacity and physical-write ordinal; bounded body/statistics/job facts; non-EOF body errors remain unknown. | +| VerifyRun | Existing strict bounded REST reader rejects read errors, duplicate keys and mismatched approved run fields before effects. | +| ACK | Exact captured queue plus message ID, one physical DELETE and status 204 before recording success. | +| Acquisition | Existing strict bounded `acquire` reader compares the one-shot request, status 200, count and IDs. | +| Session close | SDK close remains a status-only 204 effect; the response-budget transport blocks refresh PATCH, and no response body is decoded or persisted. | + +No remaining SDK lossy JSON decode or transport read-error bypass was found in +the evidence-bearing set, runner, session, poll, VerifyRun or acquisition +boundaries. The session-close response carries no evidence-bearing body and is +not used to infer drain state; production persistence/reconciliation remains +out of scope. Unknown poll/session/ACK states retain the existing journal +reservation and quarantine markers, including when a remote session was +created before the uncertainty was discovered; ACK-before-acquisition remains +unchanged. Prior six P1 corrections and replay/ordinal fixes remain in place. + +Green verification was: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainBindsACKToPhysicalDelete|TestPinnedSDKDrainRejectsAmbiguousRunnerSnapshot|TestPinnedSDKDrainRejectsAmbiguousSessionResponse' -count=1 -v +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrain|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrain|TestSecurityReview.*Drain|TestReplay.*Drain|TestFileJournal.*Drain)' -count=1 -v -timeout=180s +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrain|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrain|TestSecurityReview.*Drain|TestReplay.*Drain|TestFileJournal.*Drain)' -count=1 -timeout=180s +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +cd ../.. +bash scripts/gofmt.sh check +git diff --check +bash scripts/check-offline-experiments.sh +``` + +All commands passed: the four pinned-SDK regressions, wider normal suite, +wider race suite, vet, formatting, diff check, and both-module offline gate. +The tests used only bounded loopback fixtures; no live GitHub workflow, +runner, App credential, Keychain, launchd, Docker/Lima or cleanup operation was +performed. Rollback is a focused normal `git revert --no-edit +19f53d4e7af9578c7719c7897aa032124df6df6b` plus a separate revert of this +documentation append if needed; retain any journal, reservation and +uncertainty for inspection, and never reset, erase, replay or run live cleanup. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an From 64c6fce0f9f86ece48a7b28ee51b1aef9774f2cb Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 08:12:50 +0900 Subject: [PATCH 22/63] fix(g01): gate drain release on completed withdrawal --- .../livecanary/baseline_message.go | 2 +- experiments/g01-scaleset/livecanary/drain.go | 67 +++++- .../livecanary/drain_codex_followup_test.go | 203 ++++++++++++++++++ 3 files changed, 261 insertions(+), 11 deletions(-) create mode 100644 experiments/g01-scaleset/livecanary/drain_codex_followup_test.go diff --git a/experiments/g01-scaleset/livecanary/baseline_message.go b/experiments/g01-scaleset/livecanary/baseline_message.go index 41532ff..c441b8c 100644 --- a/experiments/g01-scaleset/livecanary/baseline_message.go +++ b/experiments/g01-scaleset/livecanary/baseline_message.go @@ -79,7 +79,7 @@ func decodeBaselineRunner(data []byte) (*baselineRunnerFacts, error) { ScaleSetID *int `json:"runnerScaleSetId"` } `json:"value"` } - if DecodeStrict(data, &w) != nil || w.Count == nil || w.Value == nil || *w.Count < 0 || *w.Count > 1 || *w.Count != len(*w.Value) { + if !uniqueKeys(json.NewDecoder(strings.NewReader(string(data)))) || json.Unmarshal(data, &w) != nil || w.Count == nil || w.Value == nil || *w.Count < 0 || *w.Count > 1 || *w.Count != len(*w.Value) { return nil, ErrRemote } facts := &baselineRunnerFacts{Count: *w.Count} diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index b780d10..f0e1224 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -323,14 +323,16 @@ func (o *drainObservation) poll(index int) *drainPollObservation { // request. The first response body is held before it reaches the SDK parser so // the listener's capacity transition and ACK/acquisition order remain visible. type drainPollHook struct { - inner http.RoundTripper - target string - wrote chan struct{} - response chan struct{} - release chan struct{} + inner http.RoundTripper + target string + wrote chan struct{} + withdrawalDone chan struct{} + response chan struct{} + release chan struct{} mu sync.Mutex wroteRequest bool + withdrawalCompleted bool responseBeforeWrite bool responseHeld bool invalid bool @@ -344,12 +346,13 @@ type drainPollHook struct { wroteCallbacks [3]int onRequestWritten func() wroteOnce sync.Once + withdrawalOnce sync.Once responseOnce sync.Once releaseOnce sync.Once } func newDrainPollHook(target string) *drainPollHook { - return &drainPollHook{inner: http.DefaultTransport, target: target, wrote: make(chan struct{}), response: make(chan struct{}), release: make(chan struct{})} + return &drainPollHook{inner: http.DefaultTransport, target: target, wrote: make(chan struct{}), withdrawalDone: make(chan struct{}), response: make(chan struct{}), release: make(chan struct{})} } func (h *drainPollHook) matches(req *http.Request) bool { @@ -548,6 +551,7 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { func (h *drainPollHook) tracedRequest(req *http.Request, attempt int) *http.Request { trace := &httptrace.ClientTrace{WroteRequest: func(info httptrace.WroteRequestInfo) { notify := false + var onRequestWritten func() h.mu.Lock() if attempt < 1 || attempt >= len(h.wroteCallbacks) { h.invalid = true @@ -562,12 +566,23 @@ func (h *drainPollHook) tracedRequest(req *http.Request, attempt int) *http.Requ h.invalid = true } else if attempt == 1 { h.wroteRequest = true + if notify { + onRequestWritten = h.onRequestWritten + } } h.mu.Unlock() - if notify && info.Err == nil && h.onRequestWritten != nil { - h.onRequestWritten() + if notify { + if info.Err == nil && onRequestWritten != nil { + onRequestWritten() + h.mu.Lock() + h.withdrawalCompleted = true + h.mu.Unlock() + } + if info.Err == nil { + h.withdrawalOnce.Do(func() { close(h.withdrawalDone) }) + } + h.wroteOnce.Do(func() { close(h.wrote) }) } - h.wroteOnce.Do(func() { close(h.wrote) }) }} return req.WithContext(httptrace.WithClientTrace(req.Context(), trace)) } @@ -731,11 +746,29 @@ func (h *drainPollHook) releaseResponse() { h.releaseOnce.Do(func() { close(h.release) }) } +func (h *drainPollHook) waitForWithdrawal(ctx context.Context) bool { + if h == nil { + return false + } + h.mu.Lock() + pending := h.wroteRequest && !h.withdrawalCompleted && !h.responseBeforeWrite + h.mu.Unlock() + if !pending { + return true + } + select { + case <-h.withdrawalDone: + return true + case <-ctx.Done(): + return false + } +} + func (h *drainPollHook) boundary() (string, bool, bool) { h.mu.Lock() defer h.mu.Unlock() switch { - case h.wroteRequest && h.responseHeld && !h.responseBeforeWrite: + case h.wroteRequest && h.withdrawalCompleted && h.responseHeld && !h.responseBeforeWrite: return drainBoundaryRequestWritten, true, !h.invalid && h.wroteCallbacks[1] == 1 case h.responseBeforeWrite: return drainBoundaryResponseBeforeWrite, h.responseHeld, false @@ -1049,7 +1082,9 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho if err != nil { return obs, ErrQuarantine } + hook.mu.Lock() hook.onRequestWritten = func() { l.SetMaxRunners(drainWithdrawnCapacity) } + hook.mu.Unlock() runCtx, stop := context.WithCancel(ctx) defer stop() c.bindDrainContext(runCtx) @@ -1078,6 +1113,12 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho // letting select's choice turn a valid run into a flaky skip. boundary, held, proven := hook.boundary() obs.Boundary, obs.ResponseHeld = boundary, held + if !proven && !hook.waitForWithdrawal(ctx) { + cancelAndJoin() + return obs, ErrQuarantine + } + boundary, held, proven = hook.boundary() + obs.Boundary, obs.ResponseHeld = boundary, held hook.releaseResponse() release = true if !proven { @@ -1105,6 +1146,12 @@ func runDrainListener(ctx context.Context, client listener.Client, setID int, ho case <-hook.response: boundary, held, proven := hook.boundary() obs.Boundary, obs.ResponseHeld = boundary, held + if !proven && !hook.waitForWithdrawal(ctx) { + cancelAndJoin() + return obs, ErrQuarantine + } + boundary, held, proven = hook.boundary() + obs.Boundary, obs.ResponseHeld = boundary, held if !proven { obs.Outcome = drainOutcomeInconclusive } diff --git a/experiments/g01-scaleset/livecanary/drain_codex_followup_test.go b/experiments/g01-scaleset/livecanary/drain_codex_followup_test.go new file mode 100644 index 0000000..e86f30a --- /dev/null +++ b/experiments/g01-scaleset/livecanary/drain_codex_followup_test.go @@ -0,0 +1,203 @@ +package livecanary + +import ( + "context" + "errors" + "io" + "net/http" + "net/http/httptrace" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/actions/scaleset" + "github.com/google/uuid" +) + +func TestPinnedSDKDrainAcceptsUnrelatedRunnerMetadata(t *testing.T) { + a := approval() + runnerBody := `{"count":1,"value":[{"id":19,"name":"` + a.workerName() + `","runnerScaleSetId":7,"status":"online","version":"2.321.0","osDescription":"fixture"}],"status":"online","version":"2.321.0"}` + _, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{runnerBody: runnerBody}) + d := &Driver{Approval: a, Journal: &memoryJournal{}, API: fixtureSDK{base}} + snapshot, err := d.drainSnapshot(context.Background(), 7, "before") + if err != nil { + t.Fatalf("runner metadata snapshot = %v, want accepted bounded identity", err) + } + if snapshot.Runner == nil || snapshot.Runner.ID != 19 || snapshot.Runner.Name != a.workerName() || snapshot.Runner.ScaleSetID != 7 { + t.Fatalf("runner metadata snapshot = %+v, want bounded runner identity", snapshot.Runner) + } +} + +type blockedWithdrawalTransport struct { + entered chan struct{} + startWrite chan struct{} + callbackStarted <-chan struct{} + effects chan string + polls atomic.Int32 + enteredOnce sync.Once +} + +func (t *blockedWithdrawalTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req.Method != http.MethodGet { + switch req.Method { + case http.MethodDelete: + t.effects <- "ack" + return &http.Response{StatusCode: http.StatusNoContent, Status: http.StatusText(http.StatusNoContent), Body: http.NoBody}, nil + case http.MethodPost: + t.effects <- "acquire" + return &http.Response{StatusCode: http.StatusOK, Status: http.StatusText(http.StatusOK), Body: io.NopCloser(strings.NewReader(`{"count":1,"value":[11]}`))}, nil + default: + return &http.Response{StatusCode: http.StatusNotFound, Status: http.StatusText(http.StatusNotFound), Body: http.NoBody}, nil + } + } + + poll := t.polls.Add(1) + trace := httptrace.ContextClientTrace(req.Context()) + if trace == nil || trace.WroteRequest == nil { + return nil, errors.New("missing poll trace") + } + if poll == 1 { + t.enteredOnce.Do(func() { close(t.entered) }) + <-t.startWrite + go trace.WroteRequest(httptrace.WroteRequestInfo{}) + select { + case <-t.callbackStarted: + case <-time.After(time.Second): + return nil, errors.New("withdrawal callback did not start") + } + return &http.Response{ + StatusCode: http.StatusOK, + Status: http.StatusText(http.StatusOK), + Body: io.NopCloser(strings.NewReader(`{"messageId":7,"messageType":"RunnerScaleSetJobMessages","statistics":{"totalAvailableJobs":1,"totalAcquiredJobs":0,"totalAssignedJobs":1,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1},"body":"[{\"messageType\":\"JobAvailable\",\"runnerRequestId\":11}]"}`)), + }, nil + } + trace.WroteRequest(httptrace.WroteRequestInfo{}) + return &http.Response{ + StatusCode: http.StatusAccepted, + Status: http.StatusText(http.StatusAccepted), + Body: io.NopCloser(strings.NewReader(`{"statistics":{"totalAvailableJobs":0,"totalAcquiredJobs":0,"totalAssignedJobs":0,"totalRunningJobs":0,"totalRegisteredRunners":1,"totalBusyRunners":0,"totalIdleRunners":1}}`)), + }, nil +} + +type blockedDrainResult struct { + observation drainObservation + err error +} + +func newBlockedWithdrawalDrain(t *testing.T, ctx context.Context) (*blockedWithdrawalTransport, *drainPollHook, <-chan struct{}, <-chan struct{}, chan<- struct{}, <-chan blockedDrainResult) { + t.Helper() + const target = "http://fixture.invalid/queue" + hook := newDrainPollHook(target) + callbackStarted := make(chan struct{}) + callbackRelease := make(chan struct{}) + callbackDone := make(chan struct{}) + transport := &blockedWithdrawalTransport{ + entered: make(chan struct{}), + startWrite: make(chan struct{}), + callbackStarted: callbackStarted, + effects: make(chan string, 4), + } + hook.inner = transport + session := &drainSyntheticSession{ + client: &http.Client{Transport: hook}, + initial: scaleset.RunnerScaleSetSession{ + SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: target, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, + order: new([]string), + } + result := make(chan blockedDrainResult, 1) + go func() { + observation, err := runDrainListener(ctx, session, 7, hook) + result <- blockedDrainResult{observation: observation, err: err} + }() + select { + case <-transport.entered: + case <-time.After(time.Second): + t.Fatal("drain poll did not enter blocked transport") + } + hook.mu.Lock() + original := hook.onRequestWritten + hook.onRequestWritten = func() { + if original != nil { + original() + } + close(callbackStarted) + <-callbackRelease + close(callbackDone) + } + hook.mu.Unlock() + close(transport.startWrite) + return transport, hook, callbackStarted, callbackDone, callbackRelease, result +} + +func TestDrainListenerDoesNotReleaseBeforeWithdrawalCompletes(t *testing.T) { + transport, hook, _, callbackDone, callbackRelease, result := newBlockedWithdrawalDrain(t, context.Background()) + select { + case <-transport.callbackStarted: + case <-time.After(time.Second): + t.Fatal("withdrawal callback did not block") + } + if _, _, proven := hook.boundary(); proven { + close(callbackRelease) + <-callbackDone + <-result + t.Fatal("response-first race was proven before withdrawal completed") + } + select { + case effect := <-transport.effects: + close(callbackRelease) + <-callbackDone + <-result + t.Fatalf("%s reached the remote effect before withdrawal callback completed", effect) + case <-time.After(50 * time.Millisecond): + } + close(callbackRelease) + select { + case <-callbackDone: + case <-time.After(time.Second): + t.Fatal("withdrawal callback did not finish") + } + select { + case got := <-result: + if got.err != nil { + t.Fatalf("blocked withdrawal drain = %v, observation=%+v", got.err, got.observation) + } + case <-time.After(time.Second): + t.Fatal("blocked withdrawal drain did not finish") + } +} + +func TestDrainListenerCancellationWhileWithdrawalBlockedDoesNotDeadlock(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + transport, _, _, callbackDone, callbackRelease, result := newBlockedWithdrawalDrain(t, ctx) + select { + case <-transport.callbackStarted: + case <-time.After(time.Second): + t.Fatal("withdrawal callback did not block") + } + cancel() + select { + case got := <-result: + if !errors.Is(got.err, ErrQuarantine) { + t.Fatalf("cancelled blocked withdrawal = %v, want quarantine", got.err) + } + case <-time.After(time.Second): + close(callbackRelease) + <-callbackDone + t.Fatal("cancelled blocked withdrawal deadlocked") + } + select { + case effect := <-transport.effects: + t.Fatalf("cancelled blocked withdrawal reached %s", effect) + default: + } + close(callbackRelease) + select { + case <-callbackDone: + case <-time.After(time.Second): + t.Fatal("cancelled withdrawal callback did not finish") + } +} From cb2c1ba7c58fb5faeab6eeaa42b4564686a9c61c Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 08:15:13 +0900 Subject: [PATCH 23/63] docs(g01): record withdrawal race correction evidence --- docs/evidence/g01-idle-drain.md | 75 +++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 9d9aa10..c909cbd 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -788,6 +788,81 @@ performed. Rollback is a focused normal `git revert --no-edit documentation append if needed; retain any journal, reservation and uncertainty for inspection, and never reset, erase, replay or run live cleanup. +### Exact-head follow-up: runner metadata and withdrawal completion + +The current-head Codex detail was read against the immutable source baseline +`88d37abea8ba4d6b793c849b38bcf797f2dbb503` with the installed review wrapper +using the separated repository argument: + +```text +bash /path/to/codex-review.sh detail 72 --repo 1XP-AI/gh-runnerd +``` + +The actionable findings were [P1 unrelated runner metadata](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973755454) +and [P1 withdrawal completion race](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973755448). + +The new regressions were added before the source correction and run against +that exact baseline at `2026-09-09T23:08:11Z` UTC: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestPinnedSDKDrainAcceptsUnrelatedRunnerMetadata|TestDrainListenerDoesNotReleaseBeforeWithdrawalCompletes|TestDrainListenerCancellationWhileWithdrawalBlockedDoesNotDeadlock' -count=1 -v -timeout=30s +``` + +The command exited 1 with sanitized outcomes: the runner metadata response was +quarantined; the deterministic blocked-callback response race observed `ack` +before withdrawal completion; and the cancellation case completed without a +deadlock. No response body, token, URL or raw SDK error was recorded. + +The follow-up source/test correction was then appended as +`64c6fce0f9f86ece48a7b28ee51b1aef9774f2cb`, with no reset, rebase, amend or +force operation. `decodeBaselineRunner` now retains the recursive +case-folded duplicate-key guard while using ordinary `json.Unmarshal` for +bounded `count`/`value` facts, so unrelated status/version metadata is +tolerated without relaxing required-field or count/value identity bounds. +The drain hook records `withdrawalCompleted` only after the capacity callback +returns, closes a separate first-callback completion signal, requires that +fact for a proven boundary, and waits for it before releasing a response that +arrived first; cancellation still force-releases and joins the listener so an +in-progress callback cannot deadlock cleanup. ACK-before-acquisition, +non-EOF/duplicate/retry guards, unknown-state retention and the unproven +server-receipt category remain unchanged. + +Post-correction focused verification on that exact SHA was: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrain.*|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerDoesNotReleaseBeforeWithdrawalCompletes|TestDrainListenerCancellationWhileWithdrawalBlockedDoesNotDeadlock|TestDrainPollHookRejectsDuplicatePhysicalWrites|TestDrainPollHookRequiresOneSuccessfulWritePerPoll)$' -count=1 -timeout=180s +``` + +At `2026-09-09T23:13:11Z` UTC this passed in 0.509s, covering the new +metadata-positive and blocked-race paths plus existing pinned-SDK positive and +negative runner/session/ACK/non-EOF and capacity/retry checks. + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrain.*|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerDoesNotReleaseBeforeWithdrawalCompletes|TestDrainListenerCancellationWhileWithdrawalBlockedDoesNotDeadlock|TestDrainPollHookRejectsDuplicatePhysicalWrites|TestDrainPollHookRequiresOneSuccessfulWritePerPoll)$' -count=1 -timeout=180s +``` + +At `2026-09-09T23:13:12Z` UTC this passed in 1.614s with no race report. +The relevant package gate then passed at `2026-09-09T23:13:20Z` UTC: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=180s +GOTOOLCHAIN=go1.26.8 go vet ./livecanary +bash ../../scripts/gofmt.sh check +git -C ../.. diff --check +``` + +The full `livecanary` package passed in 28.409s; vet, formatting and diff +checks emitted no diagnostics. These remain offline loopback tests only: no +live workflow, runner, credential, App, Keychain, launchd, Docker/Lima or +cleanup operation was performed. Rollback is a focused normal +`git revert --no-edit 64c6fce0f9f86ece48a7b28ee51b1aef9774f2cb` (and a +separate documentation revert if desired), retaining journal, reservation and +uncertainty for inspection. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an From b3d45cb648965913090dcb08c3674341f76f483a Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 08:24:20 +0900 Subject: [PATCH 24/63] fix(g01): bind drain queue to approved actions host --- experiments/g01-scaleset/livecanary/sdk.go | 41 ++++++- .../livecanary/sdk_integration_test.go | 102 ++++++++++++++++-- 2 files changed, 134 insertions(+), 9 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 1bca7e2..dd9913b 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -309,16 +309,51 @@ func (a *SDKAPI) drainFindRunner(c context.Context, name string, wire *baselineW return a.client.GetRunnerByName(wire.context(c), name) } -func validDrainQueueURL(value string) bool { +func validDrainQueueURL(value string, approvedHosts []string) bool { if value == "" || !baselineText(value, 4096) { return false } u, err := url.Parse(value) - return err == nil && u.Scheme != "" && u.Host != "" && u.User == nil && u.Fragment == "" && u.EscapedPath() == u.Path + if err != nil || u.Scheme == "" || u.Host == "" || u.User != nil || u.Fragment != "" || u.EscapedPath() != u.Path || !strings.EqualFold(u.Scheme, "https") || u.Hostname() == "" { + return false + } + queuePort := u.Port() + if queuePort == "" { + queuePort = "443" + } + if port, err := strconv.Atoi(queuePort); err != nil || port <= 0 || port > 65535 { + return false + } + for _, approved := range approvedHosts { + approvedHost, approvedPort, ok := drainApprovedHostPort(approved) + if ok && strings.EqualFold(u.Hostname(), approvedHost) && queuePort == approvedPort { + return true + } + } + return false +} + +func drainApprovedHostPort(value string) (string, string, bool) { + if value == "" || strings.ContainsAny(value, "/?#@") { + return "", "", false + } + host, port := value, "443" + if strings.Contains(value, ":") { + var err error + host, port, err = net.SplitHostPort(value) + if err != nil { + return "", "", false + } + parsed, err := strconv.Atoi(port) + if host == "" || err != nil || parsed <= 0 || parsed > 65535 { + return "", "", false + } + } + return host, port, true } func validDrainSessionWire(a Approval, id int, owner string, wire *baselineSessionFacts, session scaleset.RunnerScaleSetSession) bool { - if wire == nil || session.SessionID == [16]byte{} || wire.SessionID != session.SessionID.String() || wire.Owner != owner || session.OwnerName != owner || !validDrainQueueURL(wire.queueURL) || wire.queueURL != session.MessageQueueURL || session.MessageQueueAccessToken == "" || !wire.Statistics.completeDrain() || !wire.NestedSet || wire.SetID != id || wire.SetName != owner || wire.GroupID != a.RunnerGroupID || !wire.NestedStatistics.completeDrain() || !wire.Statistics.matches(session.Statistics) { + if wire == nil || session.SessionID == [16]byte{} || wire.SessionID != session.SessionID.String() || wire.Owner != owner || session.OwnerName != owner || !validDrainQueueURL(wire.queueURL, a.ActionsHosts) || wire.queueURL != session.MessageQueueURL || session.MessageQueueAccessToken == "" || !wire.Statistics.completeDrain() || !wire.NestedSet || wire.SetID != id || wire.SetName != owner || wire.GroupID != a.RunnerGroupID || !wire.NestedStatistics.completeDrain() || !wire.Statistics.matches(session.Statistics) { return false } set := session.RunnerScaleSet diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 3a422b2..4e093c8 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -201,7 +201,7 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { } var polls, acks, acquires, closes, snapshotReads int var server *httptest.Server - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.HasSuffix(r.URL.Path, "/actions/runners/registration-token") && r.Method == http.MethodPost: w.Header().Set("Content-Type", "application/json") @@ -267,13 +267,19 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { } })) defer server.Close() + fixtureApproval := a + fixtureApproval.ActionsHosts = []string{server.Listener.Addr().String()} buildAPI := func(hook *drainPollHook) (*SDKAPI, error) { retry := retryablehttp.NewClient() retry.RetryMax = 0 retry.Logger = nil retry.HTTPClient.Timeout = time.Second - transport := http.DefaultTransport.(*http.Transport).Clone() + fixtureTransport, ok := server.Client().Transport.(*http.Transport) + if !ok { + return nil, errors.New("TLS fixture transport is not an HTTP transport") + } + transport := fixtureTransport.Clone() transport.Proxy = nil transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { if address != server.Listener.Addr().String() { @@ -294,7 +300,7 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { if err != nil { return nil, err } - return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: server.URL + "/api/v3", approval: a, options: options}, nil + return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: server.URL + "/api/v3", approval: fixtureApproval, credentials: credentials(fixtureApproval), options: options}, nil } base, err := buildAPI(nil) if err != nil { @@ -536,6 +542,81 @@ func TestPinnedSDKDrainRejectsAmbiguousSessionResponse(t *testing.T) { } } +func TestPinnedSDKDrainRequiresApprovedHTTPSQueueHost(t *testing.T) { + a := approval() + for _, tc := range []struct { + name string + queueURL string + wantOpen bool + }{ + {name: "approved fixture host", queueURL: "QUEUE_URL", wantOpen: true}, + {name: "control plane API host", queueURL: "https://api.github.com/_apis/messages/queue", wantOpen: false}, + {name: "unapproved Actions host", queueURL: "https://fixture.actions.githubusercontent.com/_apis/messages/queue", wantOpen: false}, + {name: "plain HTTP", queueURL: "http://fixture.actions.githubusercontent.com/_apis/messages/queue", wantOpen: false}, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, _, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + sessionBody: pinnedDrainSessionBody(a, tc.queueURL), + allowOpenError: !tc.wantOpen, + }) + if tc.wantOpen { + if fixture.openErr != nil || session == nil { + t.Fatalf("approved HTTPS queue rejected: err=%v session=%v", fixture.openErr, session) + } + hook.mu.Lock() + target := hook.target + hook.mu.Unlock() + if target != fixture.server.URL+"/queue" { + t.Fatalf("approved queue target = %q, want fixture queue", target) + } + return + } + if fixture.openErr == nil || session != nil { + t.Fatalf("unapproved queue accepted: err=%v session=%v", fixture.openErr, session) + } + hook.mu.Lock() + target := hook.target + hook.mu.Unlock() + if target != "" { + t.Fatalf("rejected queue assigned poll target %q", target) + } + }) + } +} + +func TestValidDrainQueueURLRequiresExactApprovedHostPort(t *testing.T) { + for _, tc := range []struct { + name string + queueURL string + approved []string + want bool + }{ + {name: "implicit HTTPS port", queueURL: "https://fixture.actions.githubusercontent.com/queue", approved: []string{"fixture.actions.githubusercontent.com"}, want: true}, + {name: "explicit default HTTPS port", queueURL: "https://fixture.actions.githubusercontent.com:443/queue", approved: []string{"fixture.actions.githubusercontent.com"}, want: true}, + {name: "wrong approved port", queueURL: "https://fixture.actions.githubusercontent.com:8443/queue", approved: []string{"fixture.actions.githubusercontent.com:9443"}, want: false}, + {name: "wrong queue port", queueURL: "https://fixture.actions.githubusercontent.com:8443/queue", approved: []string{"fixture.actions.githubusercontent.com:8443"}, want: true}, + {name: "API control plane", queueURL: "https://api.github.com/queue", approved: []string{"fixture.actions.githubusercontent.com"}, want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := validDrainQueueURL(tc.queueURL, tc.approved); got != tc.want { + t.Fatalf("queue URL validity = %v, want %v for %q and %v", got, tc.want, tc.queueURL, tc.approved) + } + }) + } +} + +func pinnedDrainSessionBody(a Approval, queueURL string) string { + data, _ := json.Marshal(map[string]any{ + "sessionId": "00000000-0000-4000-8000-000000000021", + "ownerName": a.setName(), + "runnerScaleSet": pinnedDrainScaleSet(a), + "messageQueueUrl": queueURL, + "messageQueueAccessToken": "fixture-queue", + "statistics": &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }) + return string(data) +} + func pinnedDrainJobBody(a Approval) string { data, _ := json.Marshal([]map[string]any{{ "messageType": "JobAvailable", @@ -593,7 +674,7 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options runnerBody: options.runnerBody, snapshotBodies: append([]string(nil), options.snapshotBodies...), } - fixture.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fixture.server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch { case strings.HasSuffix(r.URL.Path, "/actions/runners/registration-token") && r.Method == http.MethodPost: w.Header().Set("Content-Type", "application/json") @@ -689,13 +770,22 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options } })) t.Cleanup(fixture.server.Close) + fixtureApproval := a + // The offline TLS fixture's loopback host is explicitly approved for this + // test-only SDK factory; production Approval validation still permits only + // the provider's Actions hostnames. + fixtureApproval.ActionsHosts = []string{fixture.server.Listener.Addr().String()} buildAPI := func(hook *drainPollHook) (*SDKAPI, error) { retry := retryablehttp.NewClient() retry.RetryMax = 0 retry.Logger = nil retry.HTTPClient.Timeout = time.Second - transport := http.DefaultTransport.(*http.Transport).Clone() + fixtureTransport, ok := fixture.server.Client().Transport.(*http.Transport) + if !ok { + return nil, errors.New("TLS fixture transport is not an HTTP transport") + } + transport := fixtureTransport.Clone() transport.Proxy = nil transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { if address != fixture.server.Listener.Addr().String() { @@ -726,7 +816,7 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options if err != nil { return nil, err } - return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: fixture.server.URL + "/api/v3", approval: a, credentials: credentials(a), options: options}, nil + return &SDKAPI{client: client, rest: retry.HTTPClient, baseURL: fixture.server.URL + "/api/v3", approval: fixtureApproval, credentials: credentials(fixtureApproval), options: options}, nil } base, err := buildAPI(nil) if err != nil { From d85213a2123ed260d9dd1fc01771705bc2b96ae8 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 08:27:39 +0900 Subject: [PATCH 25/63] docs(g01): record queue host boundary evidence --- docs/evidence/g01-idle-drain.md | 97 +++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 6 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index c909cbd..acb6503 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -863,6 +863,92 @@ cleanup operation was performed. Rollback is a focused normal separate documentation revert if desired), retaining journal, reservation and uncertainty for inspection. +### Exact-head follow-up: queue destination host boundary + +The additional security P1 sent at `2026-09-09T23:06:52Z` UTC identified that +`validDrainQueueURL` accepted the control-plane `api.github.com` origin as a +session `MessageQueueURL`. Because the pinned SDK uses that URL for queue GET +and ACK DELETE requests with the session bearer, accepting the API origin could +send queue credentials outside the exact approved Actions destination. The +required policy is HTTPS plus an exact approved `Approval.ActionsHosts` +host/port pair; the control-plane API host remains permitted only by the +general control-plane transport path and is not reused for queue validation. + +The new pinned `OpenDrainSession` regression was added and run before the +production correction at `2026-09-09T23:21:25Z` UTC, with source baseline +`cb2c1ba7c58fb5faeab6eeaa42b4564686a9c61c`: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrainRequiresApprovedHTTPSQueueHost$' -count=1 -v -timeout=180s +``` + +The command exited 1 as required. The approved HTTPS fixture case passed, while +the API control-plane host, an unapproved Actions-like host and a plain HTTP +host were each accepted by the old production code; the failure also showed +that the old path would return a session instead of rejecting before assigning +the poll target. No raw response body, bearer, private path or SDK error was +recorded in this evidence. + +The minimal source/test correction was then appended and pushed as +`b3d45cb648965913090dcb08c3674341f76f483a`. `validDrainQueueURL` now requires +HTTPS, a valid bounded port, and exact host/port membership in the supplied +`Approval.ActionsHosts`; a bare approved hostname means the default HTTPS port, +while an explicit fixture host/port is accepted only when that exact pair is +listed. `validDrainSessionWire` performs this check before `OpenDrainSession` +assigns `hook.target`, so rejected queue URLs cannot become poll or ACK +destinations. The pinned loopback fixture now uses a test-only TLS server and +explicitly lists its listener host/port in its fixture approval; production +`Approval.Validate` and the general API-host transport allowlist were not +weakened or reused for queue identity. + +Focused post-fix normal verification at `2026-09-09T23:24:36Z` UTC was: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrain.*|TestValidDrainQueueURLRequiresExactApprovedHostPort|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerDoesNotReleaseBeforeWithdrawalCompletes|TestDrainListenerCancellationWhileWithdrawalBlockedDoesNotDeadlock|TestDrainPollHookRejectsDuplicatePhysicalWrites|TestDrainPollHookRequiresOneSuccessfulWritePerPoll)$' -count=1 -timeout=180s +``` + +The command passed in 0.594s. It covered the approved/rejected queue-host +matrix, unrelated runner metadata acceptance, duplicate/case-folded session +and runner identity rejection, non-EOF poll failure, physical ACK binding, +withdrawn-poll/capacity/cursor/retry fences, and the blocked withdrawal +response race plus cancellation join. + +The corresponding race verification at `2026-09-09T23:24:45Z` UTC was: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrain.*|TestValidDrainQueueURLRequiresExactApprovedHostPort|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerDoesNotReleaseBeforeWithdrawalCompletes|TestDrainListenerCancellationWhileWithdrawalBlockedDoesNotDeadlock|TestDrainPollHookRejectsDuplicatePhysicalWrites|TestDrainPollHookRequiresOneSuccessfulWritePerPoll)$' -count=1 -timeout=180s +``` + +The race command passed in 2.097s with no race report. The full relevant package +run passed at `2026-09-09T23:24:55Z` UTC in 27.336s: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=180s +``` + +At `2026-09-09T23:25:31Z` UTC, `GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`bash ../../scripts/gofmt.sh check` and `git diff --check` all exited 0 with no +diagnostics. These remain offline loopback tests only; no live workflow, +runner, credential, App, Keychain, launchd, Docker, Lima or cleanup operation +was performed, and no repeated full-repository gate was run. + +This correction does not change the unknown-session contract: an ambiguous +session-open remains `uncertain=true`, `sessionID=""`, `reserved=false`, and +`authorizePhase` refuses every subsequent non-inspect phase. No known +reservation or session ID is invented, no production persistence/recovery +redesign is introduced, and ACK-before-acquisition, unknown-state retention, +duplicate/casefold guards and the unproven server-receipt category remain +unchanged. + +Rollback is a focused normal `git revert --no-edit +b3d45cb648965913090dcb08c3674341f76f483a` followed by a separate revert of +this documentation append if needed; retain the current journal and any +uncertainty for inspection, and never reset, erase, replay or run live cleanup. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an @@ -876,9 +962,8 @@ earlier correction must be isolated, revert only the reviewed source slice for [r3966770569](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770569), [r3966770561](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966770561), or the [duplicate-key regression](https://github.com/1XP-AI/gh-runnerd/commit/6e1b2144cb92a1a53db3926dd0e924c646643dfa) -after checking dependent corrections. To roll back only the current -[r3969825423](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3969825423) -correction, revert its focused source/test/evidence commit while retaining the -current journal and owned resources for inspection; do not reset, erase or -replay the journal. No live cleanup, workflow replay, runner mutation or -rollback operation was performed. +after checking dependent corrections. To roll back only the current queue-host +correction, revert the focused `b3d45cb648965913090dcb08c3674341f76f483a` +source/test/evidence commits while retaining the current journal and owned +resources for inspection; do not reset, erase or replay the journal. No live +cleanup, workflow replay, runner mutation or rollback operation was performed. From 05b5fffa9b6798e20d5454838252dd21de412fdc Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 09:02:55 +0900 Subject: [PATCH 26/63] fix(g01): bind drain acquisition and close wire requests --- .../g01-scaleset/livecanary/baseline_wire.go | 122 ++++++++++++- .../g01-scaleset/livecanary/drain_driver.go | 10 +- .../livecanary/response_budget.go | 2 +- .../livecanary/sdk_integration_test.go | 164 ++++++++++++++++++ 4 files changed, 294 insertions(+), 4 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index fae57ce..4d8bff0 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "encoding/base64" - "github.com/actions/scaleset" "io" "net/http" "net/url" @@ -12,6 +11,8 @@ import ( "strconv" "strings" "sync" + + "github.com/actions/scaleset" ) type baselineWireKey struct{} @@ -25,6 +26,8 @@ type baselineWireCapture struct { allowedHosts []string cursor int count, status int + requestIDs []int64 + requestCount int invalid bool session *baselineSessionFacts batch *baselineBatch @@ -97,6 +100,123 @@ func baselineWireAllowedHosts(a Approval, apiHost string) []string { return hosts } +// baselineRequestCaptureTransport runs immediately above the physical +// transport. User-supplied test wrappers may mutate a request before it gets +// here, so this is the final request-side boundary before any bytes leave the +// process. Only the explicitly expected acquisition request is inspected. +type baselineRequestCaptureTransport struct{ inner http.RoundTripper } + +func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) { + c, _ := req.Context().Value(baselineWireKey{}).(*baselineWireCapture) + captured := false + if c != nil { + if err := c.captureRequest(req); err != nil { + return nil, err + } + captured = c.stage == "acquire" && c.requestObserved() + } + response, err := t.inner.RoundTrip(req) + if captured { + // The standard transport closes request bodies, but the wrapper contract + // does not require a custom inner transport to do so on every path. Clear + // the bounded forwarding copy once the synchronous RoundTrip returns. + _ = req.Body.Close() + } + return response, err +} + +func validBaselineRequestIDs(ids []int64) bool { + if len(ids) == 0 || len(ids) > 4 { + return false + } + seen := make(map[int64]struct{}, len(ids)) + for _, id := range ids { + if id <= 0 { + return false + } + if _, ok := seen[id]; ok { + return false + } + seen[id] = struct{}{} + } + return true +} + +func readBaselineRequestBody(body io.ReadCloser) ([]byte, error) { + if body == nil { + return nil, ErrRemote + } + data, readErr := io.ReadAll(io.LimitReader(body, responseBodyLimit+1)) + closeErr := body.Close() + if readErr != nil || closeErr != nil || int64(len(data)) > responseBodyLimit { + clear(data) + return nil, ErrRemote + } + return data, nil +} + +func (c *baselineWireCapture) captureRequest(req *http.Request) error { + if c == nil || c.stage != "acquire" || len(c.requestIDs) == 0 || !c.target(req) { + return nil + } + c.mu.Lock() + c.requestCount++ + if c.requestCount != 1 { + c.invalid = true + c.mu.Unlock() + if req.Body != nil { + _ = req.Body.Close() + } + return ErrRemote + } + expected := slices.Clone(c.requestIDs) + c.mu.Unlock() + + data, err := readBaselineRequestBody(req.Body) + if err != nil { + c.mu.Lock() + c.invalid = true + c.mu.Unlock() + return ErrRemote + } + var got []int64 + if DecodeStrict(data, &got) != nil || !validBaselineRequestIDs(got) || !validBaselineRequestIDs(expected) || !slices.Equal(got, expected) { + clear(data) + c.mu.Lock() + c.invalid = true + c.mu.Unlock() + return ErrRemote + } + // Keep the one bounded copy only as the request stream the pinned SDK must + // forward. The capture itself retains no body bytes or decoded payload. + replacement := &baselineRequestBody{reader: bytes.NewReader(data), data: data} + req.Body = replacement + req.ContentLength = int64(len(data)) + return nil +} + +type baselineRequestBody struct { + reader *bytes.Reader + data []byte +} + +func (b *baselineRequestBody) Read(p []byte) (int, error) { return b.reader.Read(p) } +func (b *baselineRequestBody) Close() error { + clear(b.data) + b.data = nil + b.reader.Reset(nil) + return nil +} + +func (c *baselineWireCapture) requestObserved() bool { + if c == nil { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + return c.requestCount == 1 && !c.invalid +} + // Run after the ordinary response budget has wrapped the body. Only this // explicitly marked exact request is read ahead; the SDK receives identical // bytes. Nothing is captured from bootstrap, unrelated or unmarked traffic. diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 8f34c17..e5a0c1a 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -170,7 +170,7 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] if reader, ok := c.d.API.(drainEndpointHostReader); ok { apiHost = reader.drainEndpointHost() } - wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} + wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, requestIDs: slices.Clone(ids), allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} call = wire.context(call) } got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) @@ -179,7 +179,7 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] } if wire != nil { _, _, accepted, status := wire.facts() - if !wire.observed() || status != 200 || !accepted.matches(ids) { + if !wire.requestObserved() || !wire.observed() || status != 200 || !accepted.matches(ids) { return Event{}, errors.New("acquisition response did not match the one-shot request") } } @@ -372,9 +372,15 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return runErr } closeErr := d.effect(ctx, "session-close", nil, func(call context.Context) (Event, error) { + wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID} + call = wire.context(call) if err := session.Close(call); err != nil { return Event{}, err } + _, _, _, status := wire.facts() + if !wire.observed() || status != http.StatusNoContent { + return Event{}, errors.New("session close response did not match the one-shot request") + } return Event{SessionID: sessionID}, nil }) if closeErr != nil { diff --git a/experiments/g01-scaleset/livecanary/response_budget.go b/experiments/g01-scaleset/livecanary/response_budget.go index fcadfc4..7966a20 100644 --- a/experiments/g01-scaleset/livecanary/response_budget.go +++ b/experiments/g01-scaleset/livecanary/response_budget.go @@ -15,7 +15,7 @@ var errResponseBudget = errors.New("response body budget exceeded") // response wrapper without replacing that required type or patching the SDK. // The inner clone preserves the already configured TLS/host/proxy restrictions. func withResponseBudget(transport *http.Transport, wrappers ...func(http.RoundTripper) http.RoundTripper) *http.Transport { - var inner http.RoundTripper = transport + var inner http.RoundTripper = baselineRequestCaptureTransport{inner: transport} for _, wrap := range wrappers { if wrap != nil { inner = wrap(inner) diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 4e093c8..b6432b6 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -10,6 +10,7 @@ import ( "net" "net/http" "net/http/httptest" + "slices" "strings" "sync" "sync/atomic" @@ -521,6 +522,119 @@ func TestPinnedSDKDrainBindsACKToPhysicalDelete(t *testing.T) { } } +func TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody(t *testing.T) { + a := approval() + tests := []struct { + name string + mutateBody func(*http.Request) + good bool + }{ + {name: "valid pinned SDK array", good: true}, + {name: "mismatched ID", mutateBody: replaceSDKRequestBody(`[99]`)}, + {name: "duplicate ID", mutateBody: replaceSDKRequestBody(`[41,41]`)}, + {name: "case-folded duplicate field", mutateBody: replaceSDKRequestBody(`{"RequestIDs":[99],"requestids":[41]}`)}, + {name: "malformed JSON", mutateBody: replaceSDKRequestBody(`[41`)}, + {name: "oversized body", mutateBody: replaceSDKRequestBody(strings.Repeat("0", int(responseBodyLimit)+1))}, + {name: "read error", mutateBody: func(req *http.Request) { + req.Body = &sdkRequestBodyFault{data: []byte(`[41]`), err: io.ErrUnexpectedEOF} + req.ContentLength = -1 + }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + options := pinnedDrainOptions{mutateRequest: func(req *http.Request) { + if tc.mutateBody != nil && req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/acquirejobs") { + tc.mutateBody(req) + } + }} + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), options) + journal := &memoryJournal{} + d := &Driver{Approval: a, Journal: journal, API: base} + client := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), setID: 7, hook: hook} + hook.releaseResponse() + message, err := client.GetMessage(context.Background(), 0, drainInitialCapacity) + if err != nil || message == nil { + t.Fatalf("pinned SDK setup poll = message %v err %v", message, err) + } + if err := client.DeleteMessage(context.Background(), message.MessageID); err != nil { + t.Fatalf("pinned SDK setup ACK = %v", err) + } + got, err := client.AcquireJobs(context.Background(), []int64{41}) + if tc.good { + if err != nil || !slices.Equal(got, []int64{41}) { + t.Fatalf("valid pinned SDK acquisition = %v, got %v", err, got) + } + if fixture.acquires.Load() != 1 || fixture.acquireRequestBody() != "[41]" { + t.Fatalf("valid pinned SDK request = acquires %d body %q", fixture.acquires.Load(), fixture.acquireRequestBody()) + } + return + } + if !errors.Is(err, ErrQuarantine) || got != nil { + t.Fatalf("invalid acquisition body = got %v err %v, want quarantine before forwarding", got, err) + } + if fixture.acquires.Load() != 0 { + t.Fatalf("invalid acquisition body reached loopback endpoint: acquires=%d", fixture.acquires.Load()) + } + if state := replay(journal.Events()); !state.uncertain { + t.Fatal("invalid acquisition body did not retain uncertainty") + } + }) + } +} + +func TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + for _, tc := range []struct { + name string + mutate func(*http.Request) + }{ + { + name: "wrong session", + mutate: func(req *http.Request) { + if req.Method == http.MethodDelete && strings.Contains(req.URL.Path, "/sessions/") { + req.URL.Path = strings.TrimSuffix(req.URL.Path, "/00000000-0000-0000-0000-000000000021") + "/00000000-0000-0000-0000-000000000099" + req.URL.RawPath = "" + } + }, + }, + { + name: "wrong scale set", + mutate: func(req *http.Request) { + if req.Method == http.MethodDelete && strings.Contains(req.URL.Path, "/sessions/") { + req.URL.Path = strings.Replace(req.URL.Path, "/runnerscalesets/7/", "/runnerscalesets/8/", 1) + req.URL.RawPath = "" + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{mutateRequest: tc.mutate}) + journal := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := journal.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: journal, API: fixtureSDK{base}} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("wrong session-close target = %v, want quarantine", err) + } + if fixture.closeRequests.Load() != 1 { + t.Fatalf("wrong session-close target requests = %d, want one loopback attempt", fixture.closeRequests.Load()) + } + for _, event := range journal.Events() { + if event.Kind == "result" && event.Operation == "session-close" { + t.Fatal("wrong session-close target recorded a successful result") + } + } + if state := replay(journal.Events()); !state.uncertain { + t.Fatal("wrong session-close target did not retain uncertainty") + } + }) + } +} + func TestPinnedSDKDrainRejectsAmbiguousRunnerSnapshot(t *testing.T) { a := approval() runnerBody := `{"count":1,"value":[{"id":19,"name":"` + a.workerName() + `","runnerScaleSetId":7,"RunnerScaleSetId":7}]}` @@ -655,11 +769,14 @@ type pinnedDrainFixture struct { polls atomic.Int32 acks atomic.Int32 acquires atomic.Int32 + closeRequests atomic.Int32 verifyCalls atomic.Int32 snapshotReads atomic.Int32 mu sync.Mutex cursors []string capacity []string + acquireBodies []string + closePaths []string } func newPinnedDrainSession(t *testing.T, a Approval, body string) (*pinnedDrainFixture, *SDKAPI, Session, *drainPollHook) { @@ -741,6 +858,12 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options w.WriteHeader(http.StatusNoContent) case strings.HasSuffix(r.URL.Path, "/acquirejobs") && r.Method == http.MethodPost: fixture.acquires.Add(1) + requestBody, readErr := io.ReadAll(r.Body) + if readErr == nil { + fixture.mu.Lock() + fixture.acquireBodies = append(fixture.acquireBodies, string(requestBody)) + fixture.mu.Unlock() + } w.Header().Set("Content-Type", "application/json") if fixture.acquireBody != "" { _, _ = io.WriteString(w, fixture.acquireBody) @@ -748,6 +871,10 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options } _ = json.NewEncoder(w).Encode(map[string]any{"count": 1, "value": []int64{41}}) case strings.Contains(r.URL.Path, "/sessions/") && r.Method == http.MethodDelete: + fixture.closeRequests.Add(1) + fixture.mu.Lock() + fixture.closePaths = append(fixture.closePaths, r.URL.Path) + fixture.mu.Unlock() w.WriteHeader(http.StatusNoContent) case strings.HasSuffix(r.URL.Path, "/runnerscalesets/7") && r.Method == http.MethodGet: read := int(fixture.snapshotReads.Add(1)) @@ -835,6 +962,43 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options return fixture, base, session, hook } +func replaceSDKRequestBody(body string) func(*http.Request) { + return func(req *http.Request) { + data := []byte(body) + req.Body = io.NopCloser(strings.NewReader(body)) + req.ContentLength = int64(len(data)) + } +} + +type sdkRequestBodyFault struct { + data []byte + offset int + err error +} + +func (b *sdkRequestBodyFault) Read(p []byte) (int, error) { + if b.offset >= len(b.data) { + return 0, b.err + } + n := copy(p, b.data[b.offset:]) + b.offset += n + if b.offset == len(b.data) { + return n, b.err + } + return n, nil +} + +func (b *sdkRequestBodyFault) Close() error { return nil } + +func (f *pinnedDrainFixture) acquireRequestBody() string { + f.mu.Lock() + defer f.mu.Unlock() + if len(f.acquireBodies) == 0 { + return "" + } + return f.acquireBodies[len(f.acquireBodies)-1] +} + type sdkResponseBodyFaultRoundTripper struct { inner http.RoundTripper path string From 116beda04dc2bf69280cdefc4de4ef2fef397ef3 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 09:03:51 +0900 Subject: [PATCH 27/63] docs(g01): record acquisition and close wire evidence --- docs/evidence/g01-idle-drain.md | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index acb6503..8650d31 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -949,6 +949,97 @@ b3d45cb648965913090dcb08c3674341f76f483a` followed by a separate revert of this documentation append if needed; retain the current journal and any uncertainty for inspection, and never reset, erase, replay or run live cleanup. +### Exact-head follow-up: acquisition request body and session-close wire binding + +The fresh exact-head Codex detail for PR72 was read against immutable baseline +`d85213a2123ed260d9dd1fc01771705bc2b96ae`. It identified [P1 acquisition +request-body binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3974007599) +and [P1 session-close DELETE binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3974007586). +The review command was the repository's `codex-review` detail path for PR72; +no raw review payload, token or private machine path is included here. + +The meaningful red regressions were added before the source correction and run +against the exact baseline with the real pinned `github.com/actions/scaleset +v0.4.0` SDK and an offline TLS loopback fixture: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody$' -count=1 -v -timeout=180s +``` + +At `2026-09-09T23:53:39Z` UTC this exited 1. The valid SDK-array control +passed, while mismatched, duplicate, case-folded, malformed and oversized +request bodies were forwarded and accepted by the old response-only capture; +the read-error control also reached the loopback acquisition handler +(`acquires=1`). The sanitized failing assertion was `want quarantine before +forwarding`, and no request body, bearer, URL or SDK error text was retained. + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete$' -count=1 -v -timeout=180s +``` + +At `2026-09-09T23:53:46Z` UTC this exited 1: both wrong-session and wrong- +scale-set DELETE controls returned SDK `nil` from the permissive loopback +handler, so the old direct `session.Close` path recorded success instead of +retaining uncertainty. + +The minimal source/test correction was committed as +`05b5fffa9b6798e20d5454838252dd21de412fdc`. The pinned SDK's actual acquisition +schema is a JSON array of `int64` IDs. The innermost request transport capture +receives the final request after the physical-mutation seam, reads at most the +existing response budget, rejects malformed, duplicate, case-folded, semantic +duplicate, mismatched, oversized and read-error bodies before forwarding, and +replaces a valid body with the same bounded bytes for the real SDK transport; +the forwarding copy is cleared after the synchronous request and on close. +Expected IDs are cloned into the ephemeral capture, while no raw request body, +token or response error is journaled. The response-side acquisition status and +accepted IDs remain required, and ACK-before-acquisition and unknown/replay +guards are unchanged. + +Drain session close now reuses the existing `terminal-session-close` exact +wire capture with the approved SetID/session ID and requires one matching +DELETE plus HTTP 204 before `Driver.effect` can persist a successful +`session-close` result. A wrong target that nevertheless receives 204 is +therefore recorded as unknown and leaves the session/reservation fence for +inspection; it is never retried or cleaned up automatically. + +Focused green verification completed on the corrected source: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrain.*|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=180s +``` + +At `2026-09-10T00:02:24Z` UTC this passed in 0.630s, including the valid and +negative request-body matrix, wrong set/session close paths, the existing +pinned-SDK poll/ACK/acquisition/identity/queue controls and the legitimate +observed drain. + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrain.*|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=180s +``` + +At `2026-09-10T00:02:33Z` UTC this passed in 2.019s with no race report. At +`2026-09-10T00:02:42Z` UTC, `GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +relevant-file `gofmt -l`, and `git diff --check` all exited 0; gofmt emitted no +paths. Package-wide `livecanary` normal and race runs also passed before this +commit (28.241s and 40.834s respectively). All checks remained offline TLS +loopback evidence: no live runner, workflow, credential, App, Keychain, +launchd, Docker/Lima or cleanup operation was performed. + +The remaining gap is that a client-side request write and a matching response +still do not claim server receipt or an atomic remote drain barrier. A close +target mismatch may have changed remote state before it became unknown, so the +durable fence intentionally retains the session and requires operator +inspection. G01's full live evidence gate and parent goal remain open. + +Rollback is a focused normal `git revert --no-edit +05b5fffa9b6798e20d5454838252dd21de412fdc` followed by a separate revert of +this evidence append if needed; retain the current journal, reservation and +uncertainty, and never reset, erase, replay or run live cleanup. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an From 1d25d31061b0be6dcad50a67a7263c5157b69a6f Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 09:56:19 +0900 Subject: [PATCH 28/63] fix(g01): bind acquisition and session-close wire effects --- .../g01-scaleset/livecanary/baseline_wire.go | 113 ++++++++++++-- experiments/g01-scaleset/livecanary/drain.go | 1 + .../g01-scaleset/livecanary/drain_driver.go | 11 +- .../livecanary/drain_p1_followup_test.go | 141 +++++++++++++++++ experiments/g01-scaleset/livecanary/sdk.go | 5 +- .../livecanary/sdk_integration_test.go | 142 ++++++++++++++++-- 6 files changed, 380 insertions(+), 33 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 4d8bff0..bae0fbb 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -5,6 +5,7 @@ import ( "context" "encoding/base64" "io" + "net" "net/http" "net/url" "slices" @@ -28,6 +29,7 @@ type baselineWireCapture struct { count, status int requestIDs []int64 requestCount int + origin string invalid bool session *baselineSessionFacts batch *baselineBatch @@ -41,7 +43,7 @@ func (c *baselineWireCapture) context(ctx context.Context) context.Context { return context.WithValue(ctx, baselineWireKey{}, c) } func (c *baselineWireCapture) target(r *http.Request) bool { - if r.URL.Fragment != "" || r.URL.User != nil || r.URL.EscapedPath() != r.URL.Path { + if r == nil || r.URL == nil || r.URL.Fragment != "" || r.URL.User != nil || r.URL.EscapedPath() != r.URL.Path { return false } if c.stage == "poll" || c.stage == "ack" { @@ -71,9 +73,13 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } if c.stage == "terminal-session-close" { q := r.URL.Query() - return c.sessionID != "" && r.Method == "DELETE" && strings.HasSuffix(r.URL.Path, suffix+"sessions/"+c.sessionID) && len(q) == 1 && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" + origin, ok := baselineRequestOrigin(r.URL) + return c.origin != "" && ok && origin == c.origin && c.sessionID != "" && r.Method == "DELETE" && strings.HasSuffix(r.URL.Path, suffix+"sessions/"+c.sessionID) && len(q) == 1 && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" } if c.stage == "session-open" { + if len(c.allowedHosts) > 0 && !baselineOriginAllowed(r.URL, c.allowedHosts) { + return false + } suffix += "sessions" } else if c.stage == "runner-observe" { q := r.URL.Query() @@ -92,6 +98,42 @@ func (c *baselineWireCapture) target(r *http.Request) bool { return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && len(q) == 1 && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" } +func baselineAcquireRequestCandidate(r *http.Request) bool { + return r != nil && r.URL != nil && (strings.Contains(r.URL.Path, "/_apis/runtime/runnerscalesets/") || strings.HasSuffix(r.URL.Path, "/acquirejobs")) +} + +func baselineRequestOrigin(u *url.URL) (string, bool) { + if u == nil || !strings.EqualFold(u.Scheme, "https") || u.Hostname() == "" || u.User != nil || u.Fragment != "" { + return "", false + } + port := u.Port() + if port == "" { + port = "443" + } + parsed, err := strconv.Atoi(port) + if err != nil || parsed <= 0 || parsed > 65535 { + return "", false + } + return "https://" + net.JoinHostPort(strings.ToLower(u.Hostname()), port), true +} + +func baselineOriginAllowed(u *url.URL, approvedHosts []string) bool { + origin, ok := baselineRequestOrigin(u) + if !ok { + return false + } + for _, approved := range approvedHosts { + host, port, valid := drainApprovedHostPort(approved) + if !valid { + continue + } + if origin == "https://"+net.JoinHostPort(strings.ToLower(host), port) { + return true + } + } + return false +} + func baselineWireAllowedHosts(a Approval, apiHost string) []string { hosts := slices.Clone(a.ActionsHosts) if apiHost != "" && !slices.Contains(hosts, apiHost) { @@ -108,21 +150,12 @@ type baselineRequestCaptureTransport struct{ inner http.RoundTripper } func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) { c, _ := req.Context().Value(baselineWireKey{}).(*baselineWireCapture) - captured := false if c != nil { if err := c.captureRequest(req); err != nil { return nil, err } - captured = c.stage == "acquire" && c.requestObserved() - } - response, err := t.inner.RoundTrip(req) - if captured { - // The standard transport closes request bodies, but the wrapper contract - // does not require a custom inner transport to do so on every path. Clear - // the bounded forwarding copy once the synchronous RoundTrip returns. - _ = req.Body.Close() } - return response, err + return t.inner.RoundTrip(req) } func validBaselineRequestIDs(ids []int64) bool { @@ -156,7 +189,32 @@ func readBaselineRequestBody(body io.ReadCloser) ([]byte, error) { } func (c *baselineWireCapture) captureRequest(req *http.Request) error { - if c == nil || c.stage != "acquire" || len(c.requestIDs) == 0 || !c.target(req) { + if c == nil { + return nil + } + if c.stage == "session-open" { + if !c.target(req) { + return nil + } + origin, ok := baselineRequestOrigin(req.URL) + if !ok { + return c.rejectRequest(req) + } + c.mu.Lock() + if c.origin != "" && c.origin != origin { + c.invalid = true + } + c.origin = origin + c.mu.Unlock() + return nil + } + if c.stage != "acquire" || len(c.requestIDs) == 0 { + return nil + } + if !c.target(req) { + if baselineAcquireRequestCandidate(req) { + return c.rejectRequest(req) + } return nil } c.mu.Lock() @@ -195,19 +253,46 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { return nil } +func (c *baselineWireCapture) rejectRequest(req *http.Request) error { + c.mu.Lock() + c.requestCount++ + c.invalid = true + c.mu.Unlock() + if req != nil && req.Body != nil { + _ = req.Body.Close() + } + return ErrRemote +} + type baselineRequestBody struct { + mu sync.Mutex reader *bytes.Reader data []byte } -func (b *baselineRequestBody) Read(p []byte) (int, error) { return b.reader.Read(p) } +func (b *baselineRequestBody) Read(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.reader.Read(p) +} func (b *baselineRequestBody) Close() error { + b.mu.Lock() + defer b.mu.Unlock() clear(b.data) b.data = nil b.reader.Reset(nil) return nil } +func (c *baselineWireCapture) requestOrigin() string { + if c == nil { + return "" + } + c.mu.Lock() + defer c.mu.Unlock() + return c.origin +} + func (c *baselineWireCapture) requestObserved() bool { if c == nil { return false diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index f0e1224..248cb04 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -325,6 +325,7 @@ func (o *drainObservation) poll(index int) *drainPollObservation { type drainPollHook struct { inner http.RoundTripper target string + origin string wrote chan struct{} withdrawalDone chan struct{} response chan struct{} diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index e5a0c1a..4dbc260 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -372,7 +372,16 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return runErr } closeErr := d.effect(ctx, "session-close", nil, func(call context.Context) (Event, error) { - wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID} + origin := "" + if hook != nil { + hook.mu.Lock() + origin = hook.origin + hook.mu.Unlock() + } + if origin == "" { + return Event{}, ErrQuarantine + } + wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID, origin: origin} call = wire.context(call) if err := session.Close(call); err != nil { return Event{}, err diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 9d6264c..f81713f 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -5,9 +5,11 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "strings" "testing" + "time" "github.com/actions/scaleset" ) @@ -44,6 +46,145 @@ func TestBaselineAcquireTargetIsActionsOnly(t *testing.T) { } } +func TestBaselineAcquireTargetMismatchStopsBeforeInner(t *testing.T) { + for _, target := range []string{ + "https://other.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", + "https://api.example/_apis/runtime/runnerscalesets/8/acquirejobs?api-version=6.0-preview", + "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview&extra=1", + "https://api.example/_apis/runtime/runnerscalesets/7/wrong?api-version=6.0-preview", + } { + t.Run(target, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "acquire", + setID: 7, + requestIDs: []int64{41}, + allowedHosts: []string{ + "api.example", + }, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, target, strings.NewReader("[41]")) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched acquisition target error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("mismatched acquisition target reached inner transport: calls=%d", innerCalls) + } + if capture.requestObserved() { + t.Fatal("mismatched acquisition target was marked observed") + } + }) + } +} + +func TestBaselineSessionCloseTargetRequiresExactOrigin(t *testing.T) { + capture := &baselineWireCapture{stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://actions.example:443"} + for _, tc := range []struct { + name string + target string + want bool + }{ + {name: "expected origin and dynamic path", target: "https://actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", want: true}, + {name: "wrong scheme", target: "http://actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", want: false}, + {name: "wrong host", target: "https://other.actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", want: false}, + {name: "wrong port", target: "https://actions.example:8443/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", want: false}, + } { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequest(http.MethodDelete, tc.target, nil) + if err != nil { + t.Fatal(err) + } + if got := capture.target(req); got != tc.want { + t.Fatalf("session-close target match = %v, want %v for %s", got, tc.want, tc.target) + } + }) + } +} + +func TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose(t *testing.T) { + capture := &baselineWireCapture{stage: "acquire", setID: 7, requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} + release := make(chan struct{}) + returned := make(chan struct{}) + readBody := make(chan string, 1) + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + go func() { + <-release + data, _ := io.ReadAll(req.Body) + _ = req.Body.Close() + readBody <- string(data) + }() + close(returned) + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", strings.NewReader("[41]")) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("valid acquisition transport = %v", err) + } + select { + case <-returned: + case <-time.After(time.Second): + t.Fatal("inner transport did not return") + } + close(release) + select { + case got := <-readBody: + if got != "[41]" { + t.Fatalf("asynchronous forwarding body = %q, want valid request bytes", got) + } + case <-time.After(time.Second): + t.Fatal("asynchronous forwarding read did not finish") + } +} + +func TestBaselineAcquireForwardingBodyConcurrentReadCloseIsSafe(t *testing.T) { + capture := &baselineWireCapture{stage: "acquire", setID: 7, requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} + start := make(chan struct{}) + done := make(chan struct{}, 2) + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + go func() { + <-start + buf := make([]byte, 1) + for i := 0; i < 1024; i++ { + _, _ = req.Body.Read(buf) + } + done <- struct{}{} + }() + go func() { + <-start + for i := 0; i < 1024; i++ { + _ = req.Body.Close() + } + done <- struct{}{} + }() + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", strings.NewReader("[41]")) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("valid acquisition transport = %v", err) + } + close(start) + for i := 0; i < 2; i++ { + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("concurrent request body operation did not finish") + } + } +} + func TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner(t *testing.T) { a := approval() for _, tc := range []struct { diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index dd9913b..5bceb66 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -390,17 +390,18 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook if configured == nil || configured.client == nil { return nil, ErrRemote } - wire := &baselineWireCapture{stage: "session-open", setID: id} + wire := &baselineWireCapture{stage: "session-open", setID: id, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} session, err := configured.client.MessageSessionClient(wire.context(c), id, owner, configured.options...) if err != nil { return nil, ErrRemote } sessionFacts, _, _, status := wire.facts() - if session == nil || !wire.observed() || status != http.StatusOK || !validDrainSessionWire(configured.approval, id, owner, sessionFacts, session.Session()) { + if session == nil || !wire.observed() || status != http.StatusOK || wire.requestOrigin() == "" || !validDrainSessionWire(configured.approval, id, owner, sessionFacts, session.Session()) { return nil, ErrQuarantine } hook.mu.Lock() hook.target = sessionFacts.queueURL + hook.origin = wire.requestOrigin() hook.mu.Unlock() return session, nil } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index b6432b6..0d70705 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -542,7 +542,7 @@ func TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody(t *testing.T) { } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - options := pinnedDrainOptions{mutateRequest: func(req *http.Request) { + options := pinnedDrainOptions{runtimeActionsBasePath: "/tenant/v2/", mutateRequest: func(req *http.Request) { if tc.mutateBody != nil && req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/acquirejobs") { tc.mutateBody(req) } @@ -582,6 +582,68 @@ func TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody(t *testing.T) { } } +func TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture(t *testing.T) { + a := approval() + for _, tc := range []struct { + name string + mutate func(*http.Request) + }{ + { + name: "wrong allowed host", + mutate: func(req *http.Request) { + req.URL.Host = "other.example.com" + }, + }, + { + name: "wrong scale set", + mutate: func(req *http.Request) { + req.URL.Path = strings.Replace(req.URL.Path, "/runnerscalesets/7/", "/runnerscalesets/8/", 1) + req.URL.RawPath = "" + }, + }, + { + name: "wrong query", + mutate: func(req *http.Request) { + query := req.URL.Query() + query.Set("unexpected", "1") + req.URL.RawQuery = query.Encode() + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + wrongCloseOrigin: true, + extraActionsHosts: []string{"other.example.com"}, + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/acquirejobs") { + tc.mutate(req) + } + }, + }) + journal := &memoryJournal{} + d := &Driver{Approval: a, Journal: journal, API: base} + client := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), setID: 7, hook: hook} + hook.releaseResponse() + message, err := client.GetMessage(context.Background(), 0, drainInitialCapacity) + if err != nil || message == nil { + t.Fatalf("pinned SDK setup poll = message %v err %v", message, err) + } + if err := client.DeleteMessage(context.Background(), message.MessageID); err != nil { + t.Fatalf("pinned SDK setup ACK = %v", err) + } + if got, err := client.AcquireJobs(context.Background(), []int64{41}); !errors.Is(err, ErrQuarantine) || got != nil { + t.Fatalf("mutated acquisition target = got %v err %v, want quarantine", got, err) + } + if fixture.acquires.Load() != 0 { + t.Fatalf("mutated acquisition target reached loopback endpoint: acquires=%d", fixture.acquires.Load()) + } + if state := replay(journal.Events()); !state.uncertain { + t.Fatal("mutated acquisition target did not retain uncertainty") + } + }) + } +} + func TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete(t *testing.T) { a := approval() a.Phases = append(a.Phases, "drain") @@ -635,6 +697,37 @@ func TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete(t *testing.T) { } } +func TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + wrongCloseOrigin: true, + extraActionsHosts: []string{"other.example.com"}, + runtimeActionsBasePath: "/tenant/v2/", + }) + j := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("wrong session-close origin = %v, want quarantine", err) + } + if fixture.closeRequests.Load() != 1 { + t.Fatalf("wrong session-close origin requests = %d, want one loopback attempt", fixture.closeRequests.Load()) + } + for _, event := range j.Events() { + if event.Kind == "result" && event.Operation == "session-close" { + t.Fatal("wrong session-close origin recorded a successful result") + } + } + if state := replay(j.Events()); !state.uncertain { + t.Fatal("wrong session-close origin did not retain uncertainty") + } +} + func TestPinnedSDKDrainRejectsAmbiguousRunnerSnapshot(t *testing.T) { a := approval() runnerBody := `{"count":1,"value":[{"id":19,"name":"` + a.workerName() + `","runnerScaleSetId":7,"RunnerScaleSetId":7}]}` @@ -743,17 +836,20 @@ func pinnedDrainJobBody(a Approval) string { } type pinnedDrainOptions struct { - firstPollBody string - withdrawnBody string - acquireBody string - verifyBody string - sessionBody string - runnerBody string - allowOpenError bool - acceptAnyACK bool - pollReadError error - snapshotBodies []string - mutateRequest func(*http.Request) + firstPollBody string + withdrawnBody string + acquireBody string + verifyBody string + sessionBody string + runnerBody string + allowOpenError bool + acceptAnyACK bool + pollReadError error + snapshotBodies []string + mutateRequest func(*http.Request) + wrongCloseOrigin bool + extraActionsHosts []string + runtimeActionsBasePath string } type pinnedDrainFixture struct { @@ -801,7 +897,7 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options claims, _ := json.Marshal(map[string]int64{"exp": time.Now().Add(time.Hour).Unix()}) w.Header().Set("Content-Type", "application/json") _ = json.NewEncoder(w).Encode(map[string]string{ - "url": fixture.server.URL, + "url": fixture.server.URL + options.runtimeActionsBasePath, "token": "eyJhbGciOiJub25lIn0." + base64.RawURLEncoding.EncodeToString(claims) + ".", }) case strings.HasSuffix(r.URL.Path, "/sessions") && r.Method == http.MethodPost: @@ -901,7 +997,7 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options // The offline TLS fixture's loopback host is explicitly approved for this // test-only SDK factory; production Approval validation still permits only // the provider's Actions hostnames. - fixtureApproval.ActionsHosts = []string{fixture.server.Listener.Addr().String()} + fixtureApproval.ActionsHosts = append([]string{fixture.server.Listener.Addr().String()}, options.extraActionsHosts...) buildAPI := func(hook *drainPollHook) (*SDKAPI, error) { retry := retryablehttp.NewClient() @@ -915,10 +1011,14 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options transport := fixtureTransport.Clone() transport.Proxy = nil transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { - if address != fixture.server.Listener.Addr().String() { + if address != fixture.server.Listener.Addr().String() && (!options.wrongCloseOrigin || address != "other.example.com:443") { return nil, errors.New("non-fixture address denied") } - return (&net.Dialer{}).DialContext(ctx, network, address) + dialAddress := address + if address == "other.example.com:443" { + dialAddress = fixture.server.Listener.Addr().String() + } + return (&net.Dialer{}).DialContext(ctx, network, dialAddress) } wrappers := []func(http.RoundTripper) http.RoundTripper(nil) if hook != nil { @@ -932,6 +1032,16 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options return hook }) } + if options.wrongCloseOrigin { + wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { + return sdkRequestMutationRoundTripper{inner: inner, mutate: func(req *http.Request) { + if req.Method == http.MethodDelete && strings.Contains(req.URL.Path, "/sessions/") { + req.URL.Host = "other.example.com" + req.URL.RawPath = "" + } + }} + }) + } if options.mutateRequest != nil { wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { return sdkRequestMutationRoundTripper{inner: inner, mutate: options.mutateRequest} From f08e9f3e0b1b7cb5d028e437160f7afa3a838f00 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 09:57:18 +0900 Subject: [PATCH 29/63] docs(g01): record exact wire follow-up evidence --- docs/evidence/g01-idle-drain.md | 101 ++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 8650d31..fd64321 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1040,6 +1040,107 @@ Rollback is a focused normal `git revert --no-edit this evidence append if needed; retain the current journal, reservation and uncertainty, and never reset, erase, replay or run live cleanup. +### Exact-head follow-up: acquisition preflight, session-close origin, and request-body lifetime + +This follow-up was developed from immutable baseline +`116beda04dc2bf69280cdefc4de4ef2fef397ef3`. Fresh Codex detail for PR72 +identified [acquisition target preflight](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3974234277) +and [session-close endpoint origin binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3974234284) +as P1 findings. The review detail was read through the repository wrapper; no +raw review payload, token, private path, request body or SDK error was retained. + +The meaningful normal regressions were added before the source correction. At +`2026-09-10T00:32:14Z` UTC, this command was run against the immutable baseline: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run 'TestBaselineAcquireTargetMismatchStopsBeforeInner|TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose|TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin' -count=1 -v -timeout=180s +``` + +It exited 1: all four acquisition host, scale-set, query and endpoint +mutations reached the inner transport; the asynchronous forwarding control +observed an empty body after the old wrapper returned; and the wrong allowed +Actions origin was not quarantined. The failing assertions were sanitized and +no remote payload, bearer, private path or raw SDK error was recorded. + +The additional body-lifetime regression isolated the standard RoundTripper +contract race after the target/origin corrections were present. At +`2026-09-10T00:46:54Z` UTC, the new body mutex was temporarily removed from the +worktree while the target/origin corrections remained, and this race command +exited 1 with a `bytes.Reader.Reset`/`Read` data race: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^TestBaselineAcquireForwardingBodyConcurrentReadCloseIsSafe$' -count=1 -v -timeout=180s +``` + +This was a focused isolation red, not a claim that the entire pristine +baseline had been rerun. The mutex was restored immediately; no reviewer +successfully modified the owned tests or source. + +Source/test commit `1d25d31061b0be6dcad50a67a7263c5157b69a6f` makes the minimal +corrections. Acquisition-shaped non-target requests are rejected before the +inner transport, while bootstrap requests outside that shape remain +untouched; the valid acquisition target still has strict host, path, method, +query and ID/body checks. Session-open capture now validates the actual HTTPS +scheme/host/port against the approved runtime identity and records that exact +origin; terminal session close requires the same origin while retaining the +legitimate dynamic Actions API base path. The capture transport no longer +closes the replacement body after the inner RoundTripper returns, and the +bounded forwarding body's `Read` and `Close` operations are synchronized for +the asynchronous ownership permitted by `net/http.RoundTripper`. No blanket +response clean-EOF close-error quarantine was added. + +Fresh focused normal verification at `2026-09-10T00:50:24Z` UTC was: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineAcquireTargetMismatchStopsBeforeInner|TestBaselineSessionCloseTargetRequiresExactOrigin|TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose|TestBaselineAcquireForwardingBodyConcurrentReadCloseIsSafe|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin|TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody)$' -count=1 -timeout=180s +``` + +It passed in 0.529s. The corresponding focused race command at +`2026-09-10T00:50:45Z` UTC passed in 1.432s with no race report. Both suites +used the real pinned `github.com/actions/scaleset v0.4.0` SDK and offline TLS +loopback fixtures, including the legitimate dynamic `/tenant/v2/` path and a +wrong-but-otherwise-allowed close origin. + +The complete relevant package normal run began at `2026-09-10T00:51:11Z` UTC +and passed in 26.637s: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=240s +``` + +The package-wide race command was also run in the final validation window and +the captured successful result was 39.272s: + +```text +cd experiments/g01-scaleset +GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=300s +``` + +At `2026-09-10T00:53:42Z` UTC, `GOTOOLCHAIN=go1.26.8 go vet ./livecanary` exited +0. At `2026-09-10T00:53:50Z` UTC, `bash scripts/gofmt.sh check` exited 0; +`git diff --check` also exited 0. No additional root-plus-G01/G02 offline gate +was rerun after the coordinator's throughput direction; those exact broader +gates remain coordinator/CI evidence. All work here stayed offline: no live +runner, workflow, credential, App, Keychain, launchd, Docker, Lima, ScaleSet, +JIT or cleanup operation was performed. + +The bounded security adjudication independently confirmed the standard HTTP +early-response/body-truncation race and found no actionable clean-EOF response +close-error contract issue. Final independent delta review, exact-head Codex +review and CI remain coordinator gates; this worker does not claim those gates +are complete. The durable unknown/replay fence remains unchanged whenever a +target or response cannot be proven, and the live G01 evidence gate and parent +goal remain open. + +Rollback is a focused normal `git revert --no-edit +1d25d31061b0be6dcad50a67a7263c5157b69a6f` followed by a separate revert of +this evidence append if needed; retain the current journal, reservation and +uncertainty, and never reset, erase, replay or run live cleanup. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an From c0e8a2ec7021b1903ef13dec4033ba4d8840f69b Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 15:16:45 +0900 Subject: [PATCH 30/63] fix(g01): fence marked session-open target mismatches --- docs/evidence/g01-idle-drain.md | 55 +++++++++++++++++++ .../g01-scaleset/livecanary/baseline_wire.go | 7 +++ .../livecanary/drain_p1_followup_test.go | 50 +++++++++++++++++ 3 files changed, 112 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index fd64321..9161c7a 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1141,6 +1141,61 @@ Rollback is a focused normal `git revert --no-edit this evidence append if needed; retain the current journal, reservation and uncertainty, and never reset, erase, replay or run live cleanup. +### Exact-head follow-up: fail-closed marked session-open target binding + +Fresh exact-head Codex detail for PR72 identified [P1 session-open target +pre-forwarding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3974562078) +on `f08e9f3e0b1b7cb5d028e437160f7afa3a838f00`. The finding covers marked +session-open host, path, scale-set ID, method and query mutations: the old +capture returned success for a non-target request, allowing the innermost +transport to run before `OpenDrainSession` rejected the response and discarded +the remote session identity. + +The meaningful red regression was added before the source correction: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestBaselineSessionOpenTargetMismatchStopsBeforeInner$' -count=1 -v -timeout=60s +``` + +It exited 1. The valid session-open and both SDK bootstrap controls passed, while +the wrong-host, wrong-scale-set, wrong-endpoint-path, wrong-method and wrong- +query cases returned nil instead of rejecting before the inner transport; the +counter assertions therefore exposed the pre-forwarding gap. No request body, +token, URL, response error or private path was retained. + +The minimal source/test correction adds a narrow session-open candidate check to +the marked capture: requests in the `/runnerscalesets/` route family that fail +the exact host/path/method/query target are rejected before the inner transport. +The two pinned SDK bootstrap POST paths remain unclassified and continue through +the transport, while the valid session-open request remains accepted. Existing +acquisition target preflight, session-close origin binding and asynchronous +request-body lifetime fixes are unchanged. + +The batched focused normal verification was: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineSessionOpenTargetMismatchStopsBeforeInner|TestBaselineAcquireTargetMismatchStopsBeforeInner|TestBaselineSessionCloseTargetRequiresExactOrigin|TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose|TestBaselineAcquireForwardingBodyConcurrentReadCloseIsSafe|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin|TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody|TestPinnedSDKDrainRejectsAmbiguousSessionResponse|TestPinnedSDKDrainRequiresApprovedHTTPSQueueHost|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=180s +``` + +This passed. The corresponding focused race command passed with no race report; +the valid session-open/bootstrap controls, all five session-open mutation +boundaries, prior acquisition/origin/body-lifetime regressions and pinned SDK +drain integration all remained green. These are offline loopback tests only; +the full G01 live evidence gate and parent goal remain open. + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestBaselineSessionOpenTargetMismatchStopsBeforeInner|TestBaselineAcquireTargetMismatchStopsBeforeInner|TestBaselineSessionCloseTargetRequiresExactOrigin|TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose|TestBaselineAcquireForwardingBodyConcurrentReadCloseIsSafe|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin|TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody|TestPinnedSDKDrainRejectsAmbiguousSessionResponse|TestPinnedSDKDrainRequiresApprovedHTTPSQueueHost|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -timeout=180s +``` + +This passed in 1.838s with no race report. + +Rollback is a focused normal `git revert --no-edit` of this correction commit; +retain the journal, reservation and uncertainty, and never reset, erase, replay, +or run live cleanup. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index bae0fbb..8a1ba82 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -102,6 +102,10 @@ func baselineAcquireRequestCandidate(r *http.Request) bool { return r != nil && r.URL != nil && (strings.Contains(r.URL.Path, "/_apis/runtime/runnerscalesets/") || strings.HasSuffix(r.URL.Path, "/acquirejobs")) } +func baselineSessionOpenRequestCandidate(r *http.Request) bool { + return r != nil && r.URL != nil && strings.Contains(r.URL.Path, "/runnerscalesets/") +} + func baselineRequestOrigin(u *url.URL) (string, bool) { if u == nil || !strings.EqualFold(u.Scheme, "https") || u.Hostname() == "" || u.User != nil || u.Fragment != "" { return "", false @@ -194,6 +198,9 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { } if c.stage == "session-open" { if !c.target(req) { + if baselineSessionOpenRequestCandidate(req) { + return c.rejectRequest(req) + } return nil } origin, ok := baselineRequestOrigin(req.URL) diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index f81713f..8d6a8ad 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -84,6 +84,56 @@ func TestBaselineAcquireTargetMismatchStopsBeforeInner(t *testing.T) { } } +func TestBaselineSessionOpenTargetMismatchStopsBeforeInner(t *testing.T) { + for _, tc := range []struct { + name string + method string + target string + reject bool + }{ + {name: "valid session-open", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview"}, + {name: "wrong host", method: http.MethodPost, target: "https://other.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", reject: true}, + {name: "wrong scale set", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/8/sessions?api-version=6.0-preview", reject: true}, + {name: "wrong endpoint path", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/not-session?api-version=6.0-preview", reject: true}, + {name: "wrong method", method: http.MethodGet, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", reject: true}, + {name: "wrong query", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview&unexpected=1", reject: true}, + {name: "registration bootstrap", method: http.MethodPost, target: "https://api.example/fixture-org/actions/runners/registration-token"}, + {name: "actions bootstrap", method: http.MethodPost, target: "https://api.example/fixture-org/actions/runner-registration"}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{stage: "session-open", setID: 7, allowedHosts: []string{"api.example"}} + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, tc.target, strings.NewReader("{}")) + if err != nil { + t.Fatal(err) + } + _, err = transport.RoundTrip(req) + if tc.reject { + if !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched session-open target error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("mismatched session-open target reached inner transport: calls=%d", innerCalls) + } + if capture.observed() { + t.Fatal("mismatched session-open target was marked observed") + } + return + } + if err != nil { + t.Fatalf("valid or bootstrap request error = %v", err) + } + if innerCalls != 1 { + t.Fatalf("valid or bootstrap request inner calls = %d, want one", innerCalls) + } + }) + } +} + func TestBaselineSessionCloseTargetRequiresExactOrigin(t *testing.T) { capture := &baselineWireCapture{stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://actions.example:443"} for _, tc := range []struct { From 6357865735034ff326401c9d535afe5d07ba3433 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Thu, 10 Sep 2026 16:07:21 +0900 Subject: [PATCH 31/63] fix(g01): bind session origin across drain and cleanup --- docs/evidence/g01-idle-drain.md | 92 +++++++++++++++++++ .../livecanary/baseline_listener.go | 49 ++++++---- .../livecanary/baseline_listener_test.go | 2 +- .../livecanary/baseline_terminal.go | 6 +- .../g01-scaleset/livecanary/baseline_wire.go | 62 ++++++++++--- .../g01-scaleset/livecanary/drain_driver.go | 4 +- .../livecanary/drain_p1_followup_test.go | 75 ++++++++++++++- experiments/g01-scaleset/livecanary/sdk.go | 2 +- 8 files changed, 249 insertions(+), 43 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 9161c7a..d9b047f 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1214,3 +1214,95 @@ correction, revert the focused `b3d45cb648965913090dcb08c3674341f76f483a` source/test/evidence commits while retaining the current journal and owned resources for inspection; do not reset, erase or replay the journal. No live cleanup, workflow replay, runner mutation or rollback operation was performed. + +### Exact-head correction: captured origin, bootstrap boundaries, and paired close + +Date: 2026-09-10. This correction addresses the two independent private review +findings on exact head c0e8a2e: acquisition was bound to the approved-host set +rather than the session-open origin, and the marked session-open classifier +could forward route-family escapes or reject an organization named +"runnerscalesets". It also addresses the paired terminal cleanup finding: the +paired close capture omitted the origin learned during its own session-open, so +the DELETE could occur while the local close receipt remained unknown. + +The red-first chronology was: + +1. The required c0 paired regression was run first with + GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture ./livecanary + -run '^TestPairedTerminalCapturedAcknowledgementCancellation$/session$' + -count=1 -v -timeout=180s. It exited 1 with + "captured close acknowledgement/history lost"; this is the c0 failure where + the close DELETE reached the fixture but the paired capture had no origin + and recorded no close response. +2. After adding the request-boundary regression cases, the focused red command + was GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run + '^(TestBaselineSessionOpenTargetMismatchStopsBeforeInner|TestBaselineAcquireTargetRequiresCapturedSessionOrigin|TestBaselineAcquireOriginMismatchStopsBeforeInner)$' + -count=1 -v -timeout=60s. It exited 1 as expected: malformed-percent, + semicolon, duplicate, case, delimiter, and omitted-route session-open cases + were forwarded; the colliding organization bootstrap cases were rejected; a + second approved acquisition origin and a missing captured origin were + accepted; and the acquisition origin mismatch reached the inner transport. + +The minimal correction then made the following boundaries explicit. RawQuery is +parsed through the error-returning parser and requires the exact key/value set, +rejecting malformed percent escapes, semicolon syntax, duplicates, empty or +unexpected pairs. A marked session-open accepts its exact target or only the +known organization registration-token and runner-registration bootstrap paths +for the approved organization, including the hosted and /api/v3 API prefixes; +all other marked non-target requests reject before the inner transport. The +organization is carried into the capture so an organization slug equal to +"runnerscalesets" remains compatible without restoring substring matching. +Acquisition now copies the exact canonical HTTPS origin captured at session-open +and requires equality with the physical acquisition request while retaining the +approved-host check. The paired listener records that session-open origin, the +paired terminal close refuses an absent origin before invoking the SDK, and the +close capture receives the recorded origin; the drain path copies its hook +origin as well. Redirects, missing origins, and ambiguous origins remain +quarantined, and the asynchronous request-body ownership and error-redaction +behavior remains unchanged. + +The first focused green command was +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run +'^(TestBaselineSessionOpenTargetMismatchStopsBeforeInner|TestBaselineAcquireTargetMismatchStopsBeforeInner|TestBaselineAcquireTargetRequiresCapturedSessionOrigin|TestBaselineAcquireOriginMismatchStopsBeforeInner|TestBaselineSessionCloseTargetRequiresExactOrigin|TestBaselineAcquireTargetIsActionsOnly|TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose|TestBaselineAcquireForwardingBodyConcurrentReadCloseIsSafe|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin|TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody|TestPinnedSDKDrainRejectsAmbiguousSessionResponse|TestPinnedSDKDrainRequiresApprovedHTTPSQueueHost|TestDriverDrainThroughPinnedSDKAndPollHook)$' +-count=1 -timeout=180s; it exited 0. The same expression with go test -race +exited 0 in 1.873s with no race diagnostics. + +The tagged required paired matrix was then run with +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture ./livecanary +-run +'^(TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' +-count=1 -v -timeout=180s; it exited 0 in 12.247s. The pinned SDK subset +was initially run with +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run +'^(TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody|TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin|TestPinnedSDKDrainRequiresApprovedHTTPSQueueHost|TestDriverDrainThroughPinnedSDKAndPollHook)$' +-count=1 -v -timeout=180s; that run failed because the explicit bootstrap +allowlist did not yet include the valid /api/v3 prefix. After adding that known +prefix, the same command exited 0 in 0.774s. + +The expanded tagged paired command +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture ./livecanary +-run +'^(TestPairedTerminalActualJournalsFinalize|TestPairedTerminalCompletionCadenceAndReceiptSeparation|TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' +-count=1 -v -timeout=240s exited 0 in 13.518s, including actual paired +terminal finalize and cleanup. The final untagged focused normal expression +covering the same request boundaries plus all pinned SDK integration cases +exited 0 in 0.509s; its exact fourteen-test expression is recorded in the +correction report. The corresponding final untagged go test -race expression +exited 0 in 1.873s with no race diagnostics. The final tagged paired race +command used the seven-test expression above with go test -race; it exited 0 in +44.187s with no race diagnostics. + +Focused verification also passed: GOWORK=off GOTOOLCHAIN=go1.26.8 go vet +./livecanary exited 0; gofmt reported no files for the touched implementation +and test files; and git diff --check exited 0. Tests named above cover the c0 +paired close root cause, exact acquisition-origin binding, session-open +host/path/method/query and route-family pre-forward rejection, organization +collision compatibility, strict raw-query parsing, dynamic session-close +paths, physical request-body binding, terminal +capacity/eligibility/cancellation/missing-receipt behavior, and pinned SDK +drain integration. Verification was offline and focused on the permitted +package and tagged fixture; no full root, G01, G02, makecheck, GitHub, +workflow, runner, credential, App, Keychain, launchd, Docker, Lima, live +cleanup, or rollback operation was performed. Independent final review, +exact-head Codex review, CI, and the live G01 evidence gate remain +coordinator-owned, and the parent G01 goal remains active. diff --git a/experiments/g01-scaleset/livecanary/baseline_listener.go b/experiments/g01-scaleset/livecanary/baseline_listener.go index a839780..e1abc54 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener.go @@ -18,23 +18,23 @@ var errBaselineCollected = errors.New("baseline callback collection complete") // No current phase/CLI calls this. The future pair orchestrator must prove // completed pairing and host preflight before invoking this experiment slice. type baselineListener struct { - finalizer *pairedBaselineScope - finalizing bool - pairGuard func() error - mu sync.Mutex - ctx context.Context - cancel context.CancelFunc - approval Approval - journal *FileJournal - api SDKAPI - identity controllerJournalIdentity - creation controllerRecordRef - setID int - session Session - initial scaleset.RunnerScaleSetSession - sessionID, queue string - running, used, invalid bool - after func(context.Context, baselineAcquisition) error + finalizer *pairedBaselineScope + finalizing bool + pairGuard func() error + mu sync.Mutex + ctx context.Context + cancel context.CancelFunc + approval Approval + journal *FileJournal + api SDKAPI + identity controllerJournalIdentity + creation controllerRecordRef + setID int + session Session + initial scaleset.RunnerScaleSetSession + sessionID, queue, origin string + running, used, invalid bool + after func(context.Context, baselineAcquisition) error } func newBaselineListenerHeld(ctx context.Context, a Approval, j *FileJournal, api *SDKAPI, setID int) (*baselineListener, error) { @@ -163,7 +163,16 @@ func (b *baselineListener) finish(r baselineRecord, known bool) (controllerRecor return ref, nil } func (b *baselineListener) wire(stage string) *baselineWireCapture { - return &baselineWireCapture{stage: stage, setID: b.setID, queue: b.queue, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} + return &baselineWireCapture{stage: stage, setID: b.setID, organization: b.approval.Organization, queue: b.queue, origin: b.origin, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} +} + +func (b *baselineListener) capturedOrigin() string { + if b == nil { + return "" + } + b.mu.Lock() + defer b.mu.Unlock() + return b.origin } func (b *baselineListener) run(after func(context.Context, baselineAcquisition) error) error { if b == nil || b.ctx == nil || b.cancel == nil || !b.mu.TryLock() { @@ -238,6 +247,7 @@ func (b *baselineListener) initialize() error { session, callErr := b.api.OpenSession(c.context(ctx), b.setID, b.approval.setName()) cancel() sf, _, _, status := c.facts() + origin := c.requestOrigin() r.Session = sf r.HTTPStatus = status known = c.observed() && sf.eligible(b.approval, b.setID) && ((callErr != nil && b.ctx.Err() != nil) || (callErr == nil && session != nil)) @@ -248,6 +258,9 @@ func (b *baselineListener) initialize() error { if known && callErr == nil { known = initial.SessionID.String() == sf.SessionID && initial.OwnerName == sf.Owner && sf.Statistics.matches(initial.Statistics) && initial.MessageQueueURL != "" } + if known && origin != "" { + b.origin = origin + } if callErr == nil && session != nil && sf != nil && initial.SessionID.String() == sf.SessionID && initial.OwnerName == b.approval.setName() { b.session = session b.sessionID = sf.SessionID diff --git a/experiments/g01-scaleset/livecanary/baseline_listener_test.go b/experiments/g01-scaleset/livecanary/baseline_listener_test.go index 60476eb..dd6fee5 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener_test.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener_test.go @@ -175,7 +175,7 @@ func newBaselineFixtureWithApproval(t *testing.T, change func(string, any) any, t.Cleanup(transport.CloseIdleConnections) outer := transport.Clone() outer.RegisterProtocol("https", baselineRoundTrip(func(r *http.Request) (*http.Response, error) { - response, err := (responseBudgetTransport{inner: transport}).RoundTrip(r) + response, err := (responseBudgetTransport{inner: baselineRequestCaptureTransport{inner: transport}}).RoundTrip(r) if err == nil && f.afterResponse != nil { f.afterResponse(r, response) } diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal.go b/experiments/g01-scaleset/livecanary/baseline_terminal.go index 76f8b40..3bd6446 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal.go @@ -151,7 +151,11 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { if s.listener.session == nil || s.listener.session.Session().SessionID.String() != state.sessionID { return ErrQuarantine } - capture := &baselineWireCapture{stage: stage, setID: s.setID, sessionID: state.sessionID} + origin := s.listener.capturedOrigin() + if origin == "" { + return ErrQuarantine + } + capture := &baselineWireCapture{stage: stage, setID: s.setID, sessionID: state.sessionID, origin: origin} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) callErr = s.listener.session.Close(capture.context(ctx)) cancel() diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 8a1ba82..ecfd57d 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -21,6 +21,7 @@ type baselineWireCapture struct { mu sync.Mutex stage string setID int + organization string runnerName string sessionID string queue string // private, captured from the exact session; never journaled @@ -64,17 +65,15 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } suffix := "/runnerscalesets/" + strconv.Itoa(c.setID) + "/" if c.stage == "set-observe" || c.stage == "terminal-set" || c.stage == "terminal-set-recheck" || c.stage == "terminal-set-absence" || c.stage == "terminal-set-delete" { - q := r.URL.Query() method := "GET" if c.stage == "terminal-set-delete" { method = "DELETE" } - return r.Method == method && strings.HasSuffix(r.URL.Path, strings.TrimSuffix(suffix, "/")) && len(q) == 1 && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" + return r.Method == method && strings.HasSuffix(r.URL.Path, strings.TrimSuffix(suffix, "/")) && baselineExactAPIVersionQuery(r.URL.RawQuery) } if c.stage == "terminal-session-close" { - q := r.URL.Query() origin, ok := baselineRequestOrigin(r.URL) - return c.origin != "" && ok && origin == c.origin && c.sessionID != "" && r.Method == "DELETE" && strings.HasSuffix(r.URL.Path, suffix+"sessions/"+c.sessionID) && len(q) == 1 && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" + return c.origin != "" && ok && origin == c.origin && c.sessionID != "" && r.Method == "DELETE" && strings.HasSuffix(r.URL.Path, suffix+"sessions/"+c.sessionID) && baselineExactAPIVersionQuery(r.URL.RawQuery) } if c.stage == "session-open" { if len(c.allowedHosts) > 0 && !baselineOriginAllowed(r.URL, c.allowedHosts) { @@ -82,10 +81,10 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } suffix += "sessions" } else if c.stage == "runner-observe" { - q := r.URL.Query() - return r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/_apis/distributedtask/pools/0/agents") && len(q) == 2 && len(q["agentName"]) == 1 && q.Get("agentName") == c.runnerName && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" + return r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/_apis/distributedtask/pools/0/agents") && baselineExactQuery(r.URL.RawQuery, map[string]string{"agentName": c.runnerName, "api-version": "6.0-preview"}) } else if c.stage == "acquire" { - if len(c.allowedHosts) == 0 || !slices.Contains(c.allowedHosts, r.URL.Host) { + origin, ok := baselineRequestOrigin(r.URL) + if c.origin == "" || !ok || origin != c.origin || len(c.allowedHosts) == 0 || !baselineOriginAllowed(r.URL, c.allowedHosts) { return false } suffix = "/_apis/runtime" + suffix + "acquirejobs" @@ -94,16 +93,49 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } else { return false } - q := r.URL.Query() - return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && len(q) == 1 && len(q["api-version"]) == 1 && q.Get("api-version") == "6.0-preview" + return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && baselineExactAPIVersionQuery(r.URL.RawQuery) } func baselineAcquireRequestCandidate(r *http.Request) bool { return r != nil && r.URL != nil && (strings.Contains(r.URL.Path, "/_apis/runtime/runnerscalesets/") || strings.HasSuffix(r.URL.Path, "/acquirejobs")) } -func baselineSessionOpenRequestCandidate(r *http.Request) bool { - return r != nil && r.URL != nil && strings.Contains(r.URL.Path, "/runnerscalesets/") +func baselineExactQuery(rawQuery string, expected map[string]string) bool { + if rawQuery == "" || len(strings.Split(rawQuery, "&")) != len(expected) { + return false + } + values, err := url.ParseQuery(rawQuery) + if err != nil || len(values) != len(expected) { + return false + } + for key, want := range expected { + got, ok := values[key] + if !ok || len(got) != 1 || got[0] != want { + return false + } + } + return true +} + +func baselineExactAPIVersionQuery(rawQuery string) bool { + return baselineExactQuery(rawQuery, map[string]string{"api-version": "6.0-preview"}) +} + +func (c *baselineWireCapture) sessionOpenBootstrapTarget(r *http.Request) bool { + if c == nil || r == nil || r.URL == nil || r.URL.Fragment != "" || r.URL.User != nil || r.URL.EscapedPath() != r.URL.Path || r.Method != http.MethodPost || r.URL.RawQuery != "" || len(c.allowedHosts) == 0 || !baselineOriginAllowed(r.URL, c.allowedHosts) || c.organization == "" || !component.MatchString(c.organization) || c.organization == "." || c.organization == ".." { + return false + } + paths := make([]string, 0, 10) + for _, prefix := range []string{"", "/api/v3"} { + paths = append(paths, + prefix+"/orgs/"+c.organization+"/actions/runners/registration-token", + prefix+"/orgs/"+c.organization+"/actions/runner-registration", + prefix+"/"+c.organization+"/actions/runners/registration-token", + prefix+"/"+c.organization+"/actions/runner-registration", + prefix+"/actions/runner-registration", + ) + } + return slices.Contains(paths, r.URL.Path) } func baselineRequestOrigin(u *url.URL) (string, bool) { @@ -149,7 +181,7 @@ func baselineWireAllowedHosts(a Approval, apiHost string) []string { // baselineRequestCaptureTransport runs immediately above the physical // transport. User-supplied test wrappers may mutate a request before it gets // here, so this is the final request-side boundary before any bytes leave the -// process. Only the explicitly expected acquisition request is inspected. +// process. Only explicitly marked session-open and acquisition requests are inspected. type baselineRequestCaptureTransport struct{ inner http.RoundTripper } func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) { @@ -198,10 +230,10 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { } if c.stage == "session-open" { if !c.target(req) { - if baselineSessionOpenRequestCandidate(req) { - return c.rejectRequest(req) + if c.sessionOpenBootstrapTarget(req) { + return nil } - return nil + return c.rejectRequest(req) } origin, ok := baselineRequestOrigin(req.URL) if !ok { diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 4dbc260..d31f207 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -164,13 +164,13 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] } } c.hook.mu.Lock() - queue := c.hook.target + queue, origin := c.hook.target, c.hook.origin c.hook.mu.Unlock() apiHost := "" if reader, ok := c.d.API.(drainEndpointHostReader); ok { apiHost = reader.drainEndpointHost() } - wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, requestIDs: slices.Clone(ids), allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} + wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, requestIDs: slices.Clone(ids), origin: origin, allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} call = wire.context(call) } got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 8d6a8ad..d58b11a 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -20,6 +20,7 @@ func TestBaselineAcquireTargetIsActionsOnly(t *testing.T) { setID: 7, queue: "https://queue.example/queue?access_token=fixture", allowedHosts: []string{"api.example"}, + origin: "https://api.example:443", } for _, tc := range []struct { name string @@ -51,6 +52,9 @@ func TestBaselineAcquireTargetMismatchStopsBeforeInner(t *testing.T) { "https://other.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", "https://api.example/_apis/runtime/runnerscalesets/8/acquirejobs?api-version=6.0-preview", "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview&extra=1", + "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview&bad=%zz", + "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview;extra=1", + "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview&api-version=6.0-preview", "https://api.example/_apis/runtime/runnerscalesets/7/wrong?api-version=6.0-preview", } { t.Run(target, func(t *testing.T) { @@ -97,11 +101,26 @@ func TestBaselineSessionOpenTargetMismatchStopsBeforeInner(t *testing.T) { {name: "wrong endpoint path", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/not-session?api-version=6.0-preview", reject: true}, {name: "wrong method", method: http.MethodGet, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", reject: true}, {name: "wrong query", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview&unexpected=1", reject: true}, - {name: "registration bootstrap", method: http.MethodPost, target: "https://api.example/fixture-org/actions/runners/registration-token"}, - {name: "actions bootstrap", method: http.MethodPost, target: "https://api.example/fixture-org/actions/runner-registration"}, + {name: "malformed query escape", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview&bad=%zz", reject: true}, + {name: "duplicate query", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview&api-version=6.0-preview", reject: true}, + {name: "semicolon query", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview;unexpected=1", reject: true}, + {name: "case route family", method: http.MethodPost, target: "https://api.example/_apis/runtime/RunnerScaleSets/7/sessions?api-version=6.0-preview", reject: true}, + {name: "route family delimiter", method: http.MethodPost, target: "https://api.example/_apis/runtime/runnerscalesets7/sessions?api-version=6.0-preview", reject: true}, + {name: "route family omitted", method: http.MethodPost, target: "https://api.example/_apis/runtime/sessions?api-version=6.0-preview", reject: true}, + {name: "registration bootstrap", method: http.MethodPost, target: "https://api.example/orgs/fixture-org/actions/runners/registration-token"}, + {name: "actions bootstrap", method: http.MethodPost, target: "https://api.example/actions/runner-registration"}, + {name: "bootstrap organization collision", method: http.MethodPost, target: "https://api.example/orgs/runnerscalesets/actions/runners/registration-token"}, + {name: "bootstrap organization collision actions", method: http.MethodPost, target: "https://api.example/orgs/runnerscalesets/actions/runner-registration"}, + {name: "bootstrap wrong organization", method: http.MethodPost, target: "https://api.example/orgs/other-org/actions/runners/registration-token", reject: true}, + {name: "bootstrap extra path", method: http.MethodPost, target: "https://api.example/orgs/fixture-org/actions/runners/registration-token/extra", reject: true}, + {name: "bootstrap unexpected query", method: http.MethodPost, target: "https://api.example/orgs/fixture-org/actions/runners/registration-token?unexpected=1", reject: true}, } { t.Run(tc.name, func(t *testing.T) { - capture := &baselineWireCapture{stage: "session-open", setID: 7, allowedHosts: []string{"api.example"}} + organization := "fixture-org" + if strings.Contains(tc.name, "organization collision") { + organization = "runnerscalesets" + } + capture := &baselineWireCapture{stage: "session-open", setID: 7, organization: organization, allowedHosts: []string{"api.example"}} innerCalls := 0 transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { innerCalls++ @@ -134,6 +153,49 @@ func TestBaselineSessionOpenTargetMismatchStopsBeforeInner(t *testing.T) { } } +func TestBaselineAcquireTargetRequiresCapturedSessionOrigin(t *testing.T) { + for _, tc := range []struct { + name string + origin string + target string + want bool + }{ + {name: "captured origin", origin: "https://api.example:443", target: "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", want: true}, + {name: "second approved origin", origin: "https://api.example:443", target: "https://actions.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview"}, + {name: "missing captured origin", target: "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview"}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{stage: "acquire", setID: 7, origin: tc.origin, allowedHosts: []string{"api.example", "actions.example"}} + req, err := http.NewRequest(http.MethodPost, tc.target, nil) + if err != nil { + t.Fatal(err) + } + if got := capture.target(req); got != tc.want { + t.Fatalf("acquisition target match = %v, want %v for %s", got, tc.want, tc.target) + } + }) + } +} + +func TestBaselineAcquireOriginMismatchStopsBeforeInner(t *testing.T) { + capture := &baselineWireCapture{stage: "acquire", setID: 7, origin: "https://api.example:443", requestIDs: []int64{41}, allowedHosts: []string{"api.example", "actions.example"}} + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://actions.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", strings.NewReader("[41]")) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched acquisition origin error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("mismatched acquisition origin reached inner transport: calls=%d", innerCalls) + } +} + func TestBaselineSessionCloseTargetRequiresExactOrigin(t *testing.T) { capture := &baselineWireCapture{stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://actions.example:443"} for _, tc := range []struct { @@ -145,6 +207,9 @@ func TestBaselineSessionCloseTargetRequiresExactOrigin(t *testing.T) { {name: "wrong scheme", target: "http://actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", want: false}, {name: "wrong host", target: "https://other.actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", want: false}, {name: "wrong port", target: "https://actions.example:8443/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", want: false}, + {name: "malformed query escape", target: "https://actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview&bad=%zz", want: false}, + {name: "semicolon query", target: "https://actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview;bad=1", want: false}, + {name: "duplicate query", target: "https://actions.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview&api-version=6.0-preview", want: false}, } { t.Run(tc.name, func(t *testing.T) { req, err := http.NewRequest(http.MethodDelete, tc.target, nil) @@ -159,7 +224,7 @@ func TestBaselineSessionCloseTargetRequiresExactOrigin(t *testing.T) { } func TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose(t *testing.T) { - capture := &baselineWireCapture{stage: "acquire", setID: 7, requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} + capture := &baselineWireCapture{stage: "acquire", setID: 7, origin: "https://api.example:443", requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} release := make(chan struct{}) returned := make(chan struct{}) readBody := make(chan string, 1) @@ -197,7 +262,7 @@ func TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose(t *testing.T) } func TestBaselineAcquireForwardingBodyConcurrentReadCloseIsSafe(t *testing.T) { - capture := &baselineWireCapture{stage: "acquire", setID: 7, requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} + capture := &baselineWireCapture{stage: "acquire", setID: 7, origin: "https://api.example:443", requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} start := make(chan struct{}) done := make(chan struct{}, 2) transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 5bceb66..5d4385a 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -390,7 +390,7 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook if configured == nil || configured.client == nil { return nil, ErrRemote } - wire := &baselineWireCapture{stage: "session-open", setID: id, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} + wire := &baselineWireCapture{stage: "session-open", setID: id, organization: configured.approval.Organization, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} session, err := configured.client.MessageSessionClient(wire.context(c), id, owner, configured.options...) if err != nil { return nil, ErrRemote From 8439c12e431bb25bd229c9783119bee125b0c0bd Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 01:34:03 +0900 Subject: [PATCH 32/63] Harden G01 drain marked request origins --- docs/evidence/g01-idle-drain.md | 119 ++++++++++++++++++ .../livecanary/baseline_listener.go | 3 +- .../livecanary/baseline_terminal.go | 4 +- .../g01-scaleset/livecanary/baseline_wire.go | 94 ++++++++++++-- .../g01-scaleset/livecanary/drain_driver.go | 52 +++++--- .../livecanary/drain_p1_followup_test.go | 116 ++++++++++++++++- experiments/g01-scaleset/livecanary/sdk.go | 2 +- .../livecanary/sdk_integration_test.go | 113 ++++++++++++++++- 8 files changed, 469 insertions(+), 34 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index d9b047f..50d406b 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -244,6 +244,125 @@ client-side only, `server_receipt` is `unproven`, the real high-level pinned listener is retained, the hook is bounded to one old and one next poll, and no pre-provision, recovery, replay, `RemoveRunner` or live operation was added. +## Exact-head P1 follow-up: marked boundaries and runtime-origin binding + +Date: 2026-09-13. This section records the current exact-head follow-up against +starting snapshot `6357865735034ff326401c9d535afe5d07ba3433`. The independent +finding URLs are [acquisition mismatch pre-forwarding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976535985), +[Scale Set origin binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976535994), +[session-open body ownership](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976536000), +and [durable finding evidence](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976536008). +The durable-evidence finding specifically named the two preceding exact-head +findings, so their URLs are retained here as well: [acquisition origin](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976171401) +and [session-open route binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976171403). + +### Red-first reproductions + +After the boundary tests were added and before the corresponding source +corrections, this focused command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineAcquireTargetMismatchStopsBeforeInner|TestBaselineSessionOpenBodyMustMatchOwnerBeforeInner|TestBaselineSnapshotRequestsRequireExactOriginBeforeInner)$' -count=1 -v -timeout=60s +``` + +The current-head red result was meaningful: the case-folded and wrong-route +acquisition mutations, every invalid/ambiguous session-open body, and the +wrong-origin Scale Set and runner requests reached the inner transport instead +of being rejected. The test retained only counters and fixed error outcomes; +no request body, token, URL, response error, or private log was recorded. + +The pinned-SDK body regression was independently run before the body fix: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrainRejectsAmbiguousSessionRequestBeforeFixture$' -count=1 -v -timeout=180s +``` + +It exited 1 because all four mutated bodies (wrong owner, case-fold duplicate, +unknown field, and malformed JSON) were accepted and reached the fixture. The +pinned Scale Set origin regression was also run before its source fix: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrainRejectsSnapshotOriginMismatchBeforeEffects$' -count=1 -v -timeout=180s +``` + +It exited 1 because the second snapshot on the other runtime origin reached the +fixture/listener path. The direct boundary red test above also covered the +runner-origin mismatch; the pinned runner regression was added after that red +reproduction and then verified against the pinned SDK. + +### Corrections and resolution evidence + +Marked acquisition captures now reject every request unless the exact Actions +route, method, query, approved HTTPS origin and one expected request-ID set are +present. A marked acquisition with no IDs is itself quarantined, and there is +no acquisition bootstrap exception; case-folded route spellings, wrong route +families, duplicate/extra query pairs and second requests stop before the inner +transport. + +Marked session-open POSTs are decoded with strict unknown-field and duplicate-key +handling. The approved pinned SDK v0.4.0 request must contain the all-zero +session ID and an exact `ownerName` equal to the approved owner; absent, wrong, +case-folded, duplicate, unknown or malformed bodies are rejected before +forwarding. A rejected body never captures an origin or session identity, so a +later close cannot claim a safe identity; the pinned fixture counter proves the +ambiguous request does not reach the fixture. + +Scale Set and runner snapshot captures now require an approved HTTPS runtime +origin. The before Scale Set request learns one exact canonical origin; the +before runner, after Scale Set and after runner observations all require that +same origin. `drainSnapshotWithOrigin` quarantines incomplete wire-reader pairs, +missing endpoint host allowlists, status/fact mismatches and any changed origin. +The production `SDKAPI` implements both wire readers and the endpoint-host +reader; the no-wire branch remains only for pre-existing synthetic API tests and +does not claim a runtime origin or serve as pinned-SDK evidence. + +The focused green command after the corrections exited 0: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineAcquireTargetIsActionsOnly|TestBaselineAcquireTargetMismatchStopsBeforeInner|TestBaselineMarkedAcquireWithoutIDsStopsBeforeInner|TestBaselineSessionOpenTargetMismatchStopsBeforeInner|TestBaselineSessionOpenBodyMustMatchOwnerBeforeInner|TestBaselineSnapshotRequestsRequireExactOriginBeforeInner|TestBaselineAcquireTargetRequiresCapturedSessionOrigin|TestBaselineAcquireOriginMismatchStopsBeforeInner|TestBaselineSessionCloseTargetRequiresExactOrigin|TestPinnedSDKDrainRejectsAmbiguousSessionRequestBeforeFixture|TestPinnedSDKDrainRejectsSnapshotOriginMismatchBeforeEffects|TestPinnedSDKDrainRejectsRunnerSnapshotOriginMismatchBeforeListener|TestDriverDrainThroughPinnedSDKAndPollHook|TestPinnedSDKDrainAcceptsUnrelatedRunnerMetadata|TestPinnedSDKDrainSnapshotsRequireStrictWireFacts)$' -count=1 -timeout=240s +``` + +The command passed in 0.427s. The same expression with `go test -race` also +passed with no race diagnostics. The runner-origin pinned test specifically +observed zero listener polls, while the Scale Set-origin test observed exactly +one fixture snapshot read and no observed drain. + +The required full package gates then passed: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +``` + +This passed in 27.859s. + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=300s +``` + +This passed in 39.211s with no race diagnostics. `GOWORK=off +GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, `gofmt -d` over the seven touched +Go files, and `git diff --check` all exited 0. + +### Finding matrix and rollback + +| Finding | Reproduction and resolution | Rollback evidence | +|---|---|---| +| [r3976535985](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976535985) acquisition mismatch | Red command above; exact marked acquisition/no-ID tests now reject before inner transport. | Revert this candidate source/test/evidence commit as one unit; no live rollback was run. | +| [r3976535994](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976535994) Scale Set origin | Red command above plus pinned second-origin test; before/after Scale Set and runner snapshots now share one exact origin. | The pinned wrong-origin test quarantines before the second fixture snapshot; focused revert only. | +| [r3976536000](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976536000) session-open body | Pinned body red command above; strict owner/body validation and zero fixture opens now pass. | Ambiguity retains no close identity; focused revert only, with no live close/cleanup. | +| [r3976536008](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976536008) durable evidence | This section records all four current URLs, the two prior URLs named by the finding, red/green results, resolution and rollback scope. | Documentation is included in the same candidate commit and can be reverted with it. | +| Independently reproduced runner-origin gap | Direct runner wrong-origin red case and pinned `TestPinnedSDKDrainRejectsRunnerSnapshotOriginMismatchBeforeListener` green regression; no separate review URL was supplied. | Runner mismatch quarantines before listener effects; focused revert only. | + +No live GitHub App, runner, workflow, Docker/Lima, Keychain, launchd, network +resource, cleanup or rollback operation was performed. The live G01 gate, +independent exact-head Codex review and CI remain coordinator-owned. + Verified locally with the pinned `github.com/actions/scaleset v0.4.0` module: ```text diff --git a/experiments/g01-scaleset/livecanary/baseline_listener.go b/experiments/g01-scaleset/livecanary/baseline_listener.go index e1abc54..46682a4 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener.go @@ -163,7 +163,7 @@ func (b *baselineListener) finish(r baselineRecord, known bool) (controllerRecor return ref, nil } func (b *baselineListener) wire(stage string) *baselineWireCapture { - return &baselineWireCapture{stage: stage, setID: b.setID, organization: b.approval.Organization, queue: b.queue, origin: b.origin, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} + return &baselineWireCapture{stage: stage, setID: b.setID, organization: b.approval.Organization, owner: b.approval.setName(), queue: b.queue, origin: b.origin, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} } func (b *baselineListener) capturedOrigin() string { @@ -401,6 +401,7 @@ func (b *baselineListener) AcquireJobs(_ context.Context, ids []int64) ([]int64, return nil, err } c := b.wire("acquire") + c.requestIDs = slices.Clone(ids) ctx, cancel := context.WithTimeout(b.ctx, operationTimeout) got, callErr := b.session.AcquireJobs(c.context(ctx), slices.Clone(ids)) cancel() diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal.go b/experiments/g01-scaleset/livecanary/baseline_terminal.go index 3bd6446..7d2aacc 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal.go @@ -131,7 +131,7 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { r.Terminal.Roster = &observation known = e == nil && state.rosterMatches(&observation, func(ref controllerRecordRef) *Event { return s.event(workerRef(ref)) }) case stage == "terminal-set" || stage == "terminal-set-recheck" || stage == "terminal-set-absence": - capture := &baselineWireCapture{stage: stage, setID: s.setID} + capture := &baselineWireCapture{stage: stage, setID: s.setID, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) set, e := s.captured.GetScaleSet(capture.context(ctx), s.setID) callErr = e @@ -171,7 +171,7 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { } known = e == nil && r.Terminal.Deletion != nil && receipt.AbsenceResult != nil case stage == "terminal-set-delete": - capture := &baselineWireCapture{stage: stage, setID: s.setID} + capture := &baselineWireCapture{stage: stage, setID: s.setID, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) callErr = s.captured.DeleteScaleSet(capture.context(ctx), s.setID) cancel() diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index ecfd57d..dbbedfa 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -22,6 +22,7 @@ type baselineWireCapture struct { stage string setID int organization string + owner string runnerName string sessionID string queue string // private, captured from the exact session; never journaled @@ -65,6 +66,9 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } suffix := "/runnerscalesets/" + strconv.Itoa(c.setID) + "/" if c.stage == "set-observe" || c.stage == "terminal-set" || c.stage == "terminal-set-recheck" || c.stage == "terminal-set-absence" || c.stage == "terminal-set-delete" { + if !c.snapshotOriginAllowed(r) { + return false + } method := "GET" if c.stage == "terminal-set-delete" { method = "DELETE" @@ -81,7 +85,7 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } suffix += "sessions" } else if c.stage == "runner-observe" { - return r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/_apis/distributedtask/pools/0/agents") && baselineExactQuery(r.URL.RawQuery, map[string]string{"agentName": c.runnerName, "api-version": "6.0-preview"}) + return c.snapshotOriginAllowed(r) && r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/_apis/distributedtask/pools/0/agents") && baselineExactQuery(r.URL.RawQuery, map[string]string{"agentName": c.runnerName, "api-version": "6.0-preview"}) } else if c.stage == "acquire" { origin, ok := baselineRequestOrigin(r.URL) if c.origin == "" || !ok || origin != c.origin || len(c.allowedHosts) == 0 || !baselineOriginAllowed(r.URL, c.allowedHosts) { @@ -96,8 +100,26 @@ func (c *baselineWireCapture) target(r *http.Request) bool { return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && baselineExactAPIVersionQuery(r.URL.RawQuery) } -func baselineAcquireRequestCandidate(r *http.Request) bool { - return r != nil && r.URL != nil && (strings.Contains(r.URL.Path, "/_apis/runtime/runnerscalesets/") || strings.HasSuffix(r.URL.Path, "/acquirejobs")) +func (c *baselineWireCapture) snapshotOriginAllowed(r *http.Request) bool { + if c == nil || r == nil || r.URL == nil || len(c.allowedHosts) == 0 || !baselineOriginAllowed(r.URL, c.allowedHosts) { + return false + } + origin, ok := baselineRequestOrigin(r.URL) + if !ok { + return false + } + return c.origin == "" || c.origin == origin +} + +func snapshotRequestCandidate(r *http.Request) bool { + if r == nil || r.URL == nil { + return false + } + if r.Method == http.MethodGet { + return true + } + path := r.URL.Path + return strings.Contains(path, "/runnerscalesets/") || strings.HasSuffix(path, "/agents") } func baselineExactQuery(rawQuery string, expected map[string]string) bool { @@ -235,6 +257,13 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { } return c.rejectRequest(req) } + data, err := readBaselineRequestBody(req.Body) + req.Body = nil + if err != nil || !validBaselineSessionOpenBody(data, c.owner) { + clear(data) + return c.rejectRequest(req) + } + installBaselineRequestBody(req, data) origin, ok := baselineRequestOrigin(req.URL) if !ok { return c.rejectRequest(req) @@ -247,15 +276,40 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { c.mu.Unlock() return nil } - if c.stage != "acquire" || len(c.requestIDs) == 0 { - return nil - } - if !c.target(req) { - if baselineAcquireRequestCandidate(req) { + if c.stage == "set-observe" || c.stage == "runner-observe" { + if !snapshotRequestCandidate(req) { + return nil + } + if !c.target(req) { + return c.rejectRequest(req) + } + origin, ok := baselineRequestOrigin(req.URL) + if !ok { return c.rejectRequest(req) } + c.mu.Lock() + c.requestCount++ + if c.origin == "" { + c.origin = origin + } else if c.origin != origin { + c.invalid = true + } + valid := c.requestCount == 1 && !c.invalid + c.mu.Unlock() + if !valid { + return c.rejectRequest(req) + } + return nil + } + if c.stage != "acquire" { return nil } + if len(c.requestIDs) == 0 { + return c.rejectRequest(req) + } + if !c.target(req) { + return c.rejectRequest(req) + } c.mu.Lock() c.requestCount++ if c.requestCount != 1 { @@ -286,12 +340,30 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { } // Keep the one bounded copy only as the request stream the pinned SDK must // forward. The capture itself retains no body bytes or decoded payload. - replacement := &baselineRequestBody{reader: bytes.NewReader(data), data: data} - req.Body = replacement - req.ContentLength = int64(len(data)) + installBaselineRequestBody(req, data) return nil } +func validBaselineSessionOpenBody(data []byte, owner string) bool { + if owner == "" { + return false + } + var body struct { + SessionID *string `json:"sessionId"` + Owner *string `json:"ownerName"` + } + return DecodeStrict(data, &body) == nil && body.SessionID != nil && *body.SessionID == "00000000-0000-0000-0000-000000000000" && body.Owner != nil && *body.Owner == owner +} + +func installBaselineRequestBody(req *http.Request, data []byte) { + if req == nil { + clear(data) + return + } + req.Body = &baselineRequestBody{reader: bytes.NewReader(data), data: data} + req.ContentLength = int64(len(data)) +} + func (c *baselineWireCapture) rejectRequest(req *http.Request) error { c.mu.Lock() c.requestCount++ diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index d31f207..6eb9c78 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -195,18 +195,37 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] } func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (drainSnapshot, error) { + snapshot, _, err := d.drainSnapshotWithOrigin(ctx, setID, stage, "") + return snapshot, err +} + +func (d *Driver) drainSnapshotWithOrigin(ctx context.Context, setID int, stage, expectedOrigin string) (drainSnapshot, string, error) { var snapshot drainSnapshot var set *scaleset.RunnerScaleSet + setReader, setWire := d.API.(drainScaleSetWireReader) + runnerReader, runnerWire := d.API.(drainRunnerWireReader) + if setWire != runnerWire || (expectedOrigin != "" && !setWire) { + return drainSnapshot{}, "", ErrQuarantine + } + origin := expectedOrigin + allowedHosts := []string(nil) + if reader, ok := d.API.(drainEndpointHostReader); ok { + allowedHosts = baselineWireAllowedHosts(d.Approval, reader.drainEndpointHost()) + } if err := d.effect(ctx, "observe-owned", nil, func(call context.Context) (Event, error) { var err error - if reader, ok := d.API.(drainScaleSetWireReader); ok { - wire := &baselineWireCapture{stage: "set-observe", setID: setID} - set, err = reader.drainGetScaleSet(call, setID, wire) + if setWire { + wire := &baselineWireCapture{stage: "set-observe", setID: setID, origin: expectedOrigin, allowedHosts: allowedHosts} + set, err = setReader.drainGetScaleSet(call, setID, wire) wireSet, status := wire.setFacts() - if err != nil || !wire.observed() || status != 200 || !wireSet.eligibleForDrain(d.Approval, setID) || !wireSet.matches(set) { - return Event{}, ErrRemote + wireOrigin := wire.requestOrigin() + if err != nil || !wire.observed() || status != 200 || wireOrigin == "" || (origin != "" && wireOrigin != origin) || !wireSet.eligibleForDrain(d.Approval, setID) || !wireSet.matches(set) { + return Event{}, ErrQuarantine } + origin = wireOrigin } else { + // Synthetic API fakes used by state-machine tests have no physical + // request origin. Production SDKAPI always takes the wire branch. set, err = d.API.GetScaleSet(call, setID) } if err != nil || set == nil || set.ID != setID || set.Name != d.Approval.setName() || set.RunnerGroupID != d.Approval.RunnerGroupID || !set.RunnerSetting.DisableUpdate { @@ -227,19 +246,22 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr snapshot.StatsKnown = true return Event{ID: set.ID}, nil }); err != nil { - return drainSnapshot{}, err + return drainSnapshot{}, "", err } var runner *scaleset.RunnerReference if err := d.effect(ctx, "observe-runner", nil, func(call context.Context) (Event, error) { var err error - if reader, ok := d.API.(drainRunnerWireReader); ok { - wire := &baselineWireCapture{stage: "runner-observe", runnerName: d.Approval.workerName()} - runner, err = reader.drainFindRunner(call, d.Approval.workerName(), wire) + if runnerWire { + wire := &baselineWireCapture{stage: "runner-observe", runnerName: d.Approval.workerName(), origin: origin, allowedHosts: allowedHosts} + runner, err = runnerReader.drainFindRunner(call, d.Approval.workerName(), wire) wireRunner, status := wire.runnerFacts() - if err != nil || !wire.observed() || status != http.StatusOK || !wireRunner.matches(runner) { + wireOrigin := wire.requestOrigin() + if err != nil || !wire.observed() || status != http.StatusOK || origin == "" || wireOrigin != origin || !wireRunner.matches(runner) { return Event{}, ErrQuarantine } } else { + // Synthetic API fakes used by state-machine tests have no physical + // request origin. Production SDKAPI always takes the wire branch. runner, err = d.API.FindRunner(call, d.Approval.workerName()) } if err != nil { @@ -254,10 +276,10 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr snapshot.Runner = &drainRunnerIdentity{ID: runner.ID, Name: runner.Name, ScaleSetID: runner.RunnerScaleSetID} return Event{ID: runner.ID, DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, nil }); err != nil { - return drainSnapshot{}, err + return drainSnapshot{}, "", err } _ = stage // Stage is retained by the caller's before/after final payload. - return snapshot, nil + return snapshot, origin, nil } func validDrainIdlePrerequisite(s drainSnapshot) bool { @@ -296,7 +318,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return ErrQuarantine } phaseSequence := phaseState.drainPhaseSequence - before, err := d.drainSnapshot(ctx, setID, "before") + before, origin, err := d.drainSnapshotWithOrigin(ctx, setID, "before", "") if err != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, err)); markerErr != nil { return markerErr @@ -357,7 +379,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, runErr)); markerErr != nil { return markerErr } - after, afterErr := d.drainSnapshot(ctx, setID, "after") + after, _, afterErr := d.drainSnapshotWithOrigin(ctx, setID, "after", origin) if afterErr != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, afterErr)); markerErr != nil { return markerErr @@ -398,7 +420,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return closeErr } - after, afterErr := d.drainSnapshot(ctx, setID, "after") + after, _, afterErr := d.drainSnapshotWithOrigin(ctx, setID, "after", origin) if afterErr != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, afterErr)); markerErr != nil { return markerErr diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index d58b11a..4eea8e8 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -56,6 +56,8 @@ func TestBaselineAcquireTargetMismatchStopsBeforeInner(t *testing.T) { "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview;extra=1", "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview&api-version=6.0-preview", "https://api.example/_apis/runtime/runnerscalesets/7/wrong?api-version=6.0-preview", + "https://api.example/_apis/runtime/RunnerScaleSets/7/AcquireJobs?api-version=6.0-preview", + "https://api.example/_apis/runtime/runnersets/7/acquire?api-version=6.0-preview", } { t.Run(target, func(t *testing.T) { capture := &baselineWireCapture{ @@ -88,6 +90,33 @@ func TestBaselineAcquireTargetMismatchStopsBeforeInner(t *testing.T) { } } +func TestBaselineMarkedAcquireWithoutIDsStopsBeforeInner(t *testing.T) { + capture := &baselineWireCapture{ + stage: "acquire", + setID: 7, + origin: "https://api.example:443", + allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", strings.NewReader("[]")) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("marked acquisition without IDs error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("marked acquisition without IDs reached inner transport: calls=%d", innerCalls) + } + if capture.requestObserved() { + t.Fatal("marked acquisition without IDs was marked observed") + } +} + func TestBaselineSessionOpenTargetMismatchStopsBeforeInner(t *testing.T) { for _, tc := range []struct { name string @@ -120,13 +149,17 @@ func TestBaselineSessionOpenTargetMismatchStopsBeforeInner(t *testing.T) { if strings.Contains(tc.name, "organization collision") { organization = "runnerscalesets" } - capture := &baselineWireCapture{stage: "session-open", setID: 7, organization: organization, allowedHosts: []string{"api.example"}} + capture := &baselineWireCapture{stage: "session-open", setID: 7, organization: organization, owner: "g01-test", allowedHosts: []string{"api.example"}} innerCalls := 0 transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { innerCalls++ return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil })} - req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, tc.target, strings.NewReader("{}")) + body := "{}" + if tc.name == "valid session-open" { + body = `{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"g01-test"}` + } + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, tc.target, strings.NewReader(body)) if err != nil { t.Fatal(err) } @@ -153,6 +186,85 @@ func TestBaselineSessionOpenTargetMismatchStopsBeforeInner(t *testing.T) { } } +func TestBaselineSessionOpenBodyMustMatchOwnerBeforeInner(t *testing.T) { + for _, tc := range []struct { + name string + body string + want bool + }{ + {name: "approved owner", body: `{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"g01-test"}`, want: true}, + {name: "missing session id", body: `{"ownerName":"g01-test"}`}, + {name: "missing owner", body: `{}`}, + {name: "wrong owner", body: `{"ownerName":"foreign-owner"}`}, + {name: "case-fold duplicate owner", body: `{"ownerName":"g01-test","OwnerName":"foreign-owner"}`}, + {name: "unknown field", body: `{"ownerName":"g01-test","unexpected":1}`}, + {name: "malformed", body: `{"ownerName":"g01-test"`}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{stage: "session-open", setID: 7, owner: "g01-test", organization: "fixture-org", allowedHosts: []string{"api.example"}} + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + _, err = transport.RoundTrip(req) + if tc.want { + if err != nil || innerCalls != 1 { + t.Fatalf("approved session-open body err=%v inner calls=%d, want one forwarded request", err, innerCalls) + } + return + } + if !errors.Is(err, ErrRemote) { + t.Fatalf("ambiguous session-open body error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("ambiguous session-open body reached inner transport: calls=%d", innerCalls) + } + }) + } +} + +func TestBaselineSnapshotRequestsRequireExactOriginBeforeInner(t *testing.T) { + for _, tc := range []struct { + name string + stage string + path string + }{ + {name: "scale-set wrong origin", stage: "set-observe", path: "/tenant/v2/_apis/runtime/runnerscalesets/7"}, + {name: "runner wrong origin", stage: "runner-observe", path: "/tenant/v2/_apis/distributedtask/pools/0/agents"}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{stage: tc.stage, setID: 7, runnerName: "g01-test-worker-1", origin: "https://api.example:443", allowedHosts: []string{"api.example", "other.example"}} + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + query := "api-version=6.0-preview" + if tc.stage == "runner-observe" { + query = "agentName=g01-test-worker-1&api-version=6.0-preview" + } + method := http.MethodGet + target := "https://other.example" + tc.path + "?" + query + req, err := http.NewRequestWithContext(capture.context(context.Background()), method, target, nil) + if err != nil { + t.Fatal(err) + } + _, err = transport.RoundTrip(req) + if !errors.Is(err, ErrRemote) { + t.Fatalf("%s error = %v, want remote rejection", tc.name, err) + } + if innerCalls != 0 { + t.Fatalf("%s reached inner transport: calls=%d", tc.name, innerCalls) + } + }) + } +} + func TestBaselineAcquireTargetRequiresCapturedSessionOrigin(t *testing.T) { for _, tc := range []struct { name string diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 5d4385a..45782df 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -390,7 +390,7 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook if configured == nil || configured.client == nil { return nil, ErrRemote } - wire := &baselineWireCapture{stage: "session-open", setID: id, organization: configured.approval.Organization, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} + wire := &baselineWireCapture{stage: "session-open", setID: id, organization: configured.approval.Organization, owner: owner, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} session, err := configured.client.MessageSessionClient(wire.context(c), id, owner, configured.options...) if err != nil { return nil, ErrRemote diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 0d70705..718a2c9 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -368,6 +368,74 @@ func TestDriverDrainThroughPinnedSDKAndPollHook(t *testing.T) { } } +func TestPinnedSDKDrainRejectsSnapshotOriginMismatchBeforeEffects(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + var snapshotRequests atomic.Int32 + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + wrongCloseOrigin: false, + extraActionsHosts: []string{"other.example.com"}, + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodGet && strings.HasSuffix(req.URL.Path, "/runnerscalesets/7") && snapshotRequests.Add(1) == 2 { + req.URL.Host = "other.example.com" + req.URL.RawPath = "" + } + }, + }) + j := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + runErr := d.Run(context.Background(), "drain") + if !errors.Is(runErr, ErrQuarantine) { + t.Fatalf("snapshot origin mismatch = %v, want quarantine", runErr) + } + if got := fixture.snapshotReads.Load(); got != 1 { + t.Fatalf("mismatched after snapshot reached fixture: reads=%d, want before only", got) + } + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { + t.Fatal("mismatched snapshot origin produced an observed drain") + } + } +} + +func TestPinnedSDKDrainRejectsRunnerSnapshotOriginMismatchBeforeListener(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + var runnerRequests atomic.Int32 + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + extraActionsHosts: []string{"other.example.com"}, + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodGet && strings.HasSuffix(req.URL.Path, "/agents") && runnerRequests.Add(1) == 1 { + req.URL.Host = "other.example.com" + req.URL.RawPath = "" + } + }, + }) + j := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("runner snapshot origin mismatch = %v, want quarantine", err) + } + if got := fixture.polls.Load(); got != 0 { + t.Fatalf("mismatched runner snapshot reached listener effects: polls=%d", got) + } + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { + t.Fatal("mismatched runner snapshot origin produced an observed drain") + } + } +} + func TestPinnedSDKDrainRejectsAmbiguousEmbeddedJobIdentityBeforeEffects(t *testing.T) { a := approval() for _, test := range []struct { @@ -749,6 +817,36 @@ func TestPinnedSDKDrainRejectsAmbiguousSessionResponse(t *testing.T) { } } +func TestPinnedSDKDrainRejectsAmbiguousSessionRequestBeforeFixture(t *testing.T) { + a := approval() + for _, tc := range []struct { + name string + body string + }{ + {name: "wrong owner", body: `{"ownerName":"foreign-owner"}`}, + {name: "case-fold duplicate owner", body: `{"ownerName":"` + a.setName() + `","OwnerName":"foreign-owner"}`}, + {name: "unknown field", body: `{"ownerName":"` + a.setName() + `","unexpected":1}`}, + {name: "malformed", body: `{"ownerName":"` + a.setName() + `"`}, + } { + t.Run(tc.name, func(t *testing.T) { + fixture, _, session, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + allowOpenError: true, + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/sessions") { + replaceSDKRequestBody(tc.body)(req) + } + }, + }) + if fixture.openErr == nil || session != nil { + t.Fatalf("ambiguous session-open request accepted: err=%v session=%v", fixture.openErr, session) + } + if got := fixture.sessionOpens.Load(); got != 0 { + t.Fatalf("ambiguous session-open request reached fixture: opens=%d", got) + } + }) + } +} + func TestPinnedSDKDrainRequiresApprovedHTTPSQueueHost(t *testing.T) { a := approval() for _, tc := range []struct { @@ -866,6 +964,7 @@ type pinnedDrainFixture struct { acks atomic.Int32 acquires atomic.Int32 closeRequests atomic.Int32 + sessionOpens atomic.Int32 verifyCalls atomic.Int32 snapshotReads atomic.Int32 mu sync.Mutex @@ -901,6 +1000,7 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options "token": "eyJhbGciOiJub25lIn0." + base64.RawURLEncoding.EncodeToString(claims) + ".", }) case strings.HasSuffix(r.URL.Path, "/sessions") && r.Method == http.MethodPost: + fixture.sessionOpens.Add(1) w.Header().Set("Content-Type", "application/json") if options.sessionBody != "" { _, _ = io.WriteString(w, strings.ReplaceAll(options.sessionBody, "QUEUE_URL", fixture.server.URL+"/queue")) @@ -1011,11 +1111,20 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options transport := fixtureTransport.Clone() transport.Proxy = nil transport.DialContext = func(ctx context.Context, network, address string) (net.Conn, error) { - if address != fixture.server.Listener.Addr().String() && (!options.wrongCloseOrigin || address != "other.example.com:443") { + aliasAllowed := options.wrongCloseOrigin && address == "other.example.com:443" + if !aliasAllowed { + for _, host := range options.extraActionsHosts { + if address == host+":443" { + aliasAllowed = true + break + } + } + } + if address != fixture.server.Listener.Addr().String() && !aliasAllowed { return nil, errors.New("non-fixture address denied") } dialAddress := address - if address == "other.example.com:443" { + if aliasAllowed { dialAddress = fixture.server.Listener.Addr().String() } return (&net.Dialer{}).DialContext(ctx, network, dialAddress) From cbf711cb582ccd6a68eca13312697be21cea580a Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 02:04:17 +0900 Subject: [PATCH 33/63] fix(g01): bind drain poll approval and session origin --- experiments/g01-scaleset/livecanary/drain.go | 88 ++++++++++++++++++- .../g01-scaleset/livecanary/drain_driver.go | 11 ++- .../livecanary/drain_p1_followup_test.go | 86 ++++++++++++++++++ .../g01-scaleset/livecanary/drain_test.go | 3 + experiments/g01-scaleset/livecanary/sdk.go | 5 ++ .../livecanary/sdk_integration_test.go | 33 +++++++ 6 files changed, 222 insertions(+), 4 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 248cb04..c83653f 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -6,6 +6,7 @@ import ( "encoding/json" "errors" "io" + "net" "net/http" "net/http/httptrace" "net/url" @@ -353,7 +354,65 @@ type drainPollHook struct { } func newDrainPollHook(target string) *drainPollHook { - return &drainPollHook{inner: http.DefaultTransport, target: target, wrote: make(chan struct{}), withdrawalDone: make(chan struct{}), response: make(chan struct{}), release: make(chan struct{})} + hook := &drainPollHook{inner: http.DefaultTransport, target: target, wrote: make(chan struct{}), withdrawalDone: make(chan struct{}), response: make(chan struct{}), release: make(chan struct{})} + if origin, ok := drainPollOrigin(target); ok { + hook.origin = origin + } + return hook +} + +func newDrainPollHookWithOrigin(target, origin string) *drainPollHook { + hook := newDrainPollHook(target) + hook.mu.Lock() + hook.origin = origin + hook.mu.Unlock() + return hook +} + +// drainPollApprovalKey marks the one listener poll that the drain boundary +// permits to reach the physical transport. Other SDK requests (session-open, +// ACK and acquisition) use their own baseline wire markers and remain +// unmarked here. +type drainPollApprovalKey struct{} + +func (h *drainPollHook) markPoll(ctx context.Context) context.Context { + if h == nil || ctx == nil { + return ctx + } + return context.WithValue(ctx, drainPollApprovalKey{}, h) +} + +func drainPollOrigin(value string) (string, bool) { + // Production session responses are admitted only with HTTPS queue URLs by + // validDrainQueueURL. HTTP remains accepted here solely for the bounded + // in-process transport seams used by offline listener tests. + u, err := url.Parse(value) + if err != nil { + return "", false + } + return drainPollURLOrigin(u) +} + +func drainPollURLOrigin(u *url.URL) (string, bool) { + if u == nil || u.Scheme == "" || u.Hostname() == "" || u.User != nil || u.Fragment != "" || u.EscapedPath() != u.Path { + return "", false + } + if !strings.EqualFold(u.Scheme, "http") && !strings.EqualFold(u.Scheme, "https") { + return "", false + } + port := u.Port() + if port == "" { + if strings.EqualFold(u.Scheme, "https") { + port = "443" + } else { + port = "80" + } + } + parsed, err := strconv.Atoi(port) + if err != nil || parsed <= 0 || parsed > 65535 { + return "", false + } + return strings.ToLower(u.Scheme) + "://" + net.JoinHostPort(strings.ToLower(u.Hostname()), port), true } func (h *drainPollHook) matches(req *http.Request) bool { @@ -403,6 +462,17 @@ func (h *drainPollHook) pollRequestValid(req *http.Request, attempt int) bool { if err != nil || want.Scheme == "" || want.Host == "" || want.User != nil || want.Fragment != "" || want.EscapedPath() != want.Path || req.Method != http.MethodGet || req.URL.Scheme != want.Scheme || req.URL.Host != want.Host || req.URL.Path != want.Path || req.URL.User != nil || req.URL.Fragment != "" || req.URL.EscapedPath() != req.URL.Path { return false } + wantOrigin, wantOriginOK := drainPollOrigin(target) + requestOrigin, requestOriginOK := drainPollURLOrigin(req.URL) + if !wantOriginOK || !requestOriginOK || wantOrigin != requestOrigin { + return false + } + h.mu.Lock() + approvedOrigin := h.origin + h.mu.Unlock() + if (approvedOrigin == "" && strings.EqualFold(want.Scheme, "https")) || (approvedOrigin != "" && requestOrigin != approvedOrigin) { + return false + } values := req.Header.Values(scaleset.HeaderScaleSetMaxCapacity) wantCapacity := drainInitialCapacity if attempt == 2 { @@ -451,18 +521,27 @@ func (h *drainPollHook) rejectPoll() (*http.Response, error) { } func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { - if h == nil || h.inner == nil || !h.matches(req) { + if h == nil || h.inner == nil { if h == nil || h.inner == nil { return nil, ErrQuarantine } + } + if req == nil { + return h.rejectPoll() + } + approved, marked := req.Context().Value(drainPollApprovalKey{}).(*drainPollHook) + if !marked { return h.inner.RoundTrip(req) } + if approved != h { + return h.rejectPoll() + } h.mu.Lock() h.pollAttempts++ attempt := h.pollAttempts first := h.pollAttempts == 1 h.mu.Unlock() - if !h.pollRequestValid(req, attempt) { + if !h.matches(req) || !h.pollRequestValid(req, attempt) { return h.rejectPoll() } if !first { @@ -870,6 +949,9 @@ func (c *drainClient) GetMessage(ctx context.Context, last, capacity int) (*scal c.obs.Ordering = append(c.obs.Ordering, map[int]string{1: "poll-old", 2: "poll-zero"}[index]) c.mu.Unlock() + if c.hook != nil { + ctx = c.hook.markPoll(ctx) + } message, err := c.inner.GetMessage(ctx, last, capacity) stats, statsKnown := drainStatistics{}, false if c.hook == nil { diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 6eb9c78..e4863b1 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -63,6 +63,9 @@ func (c *journaledDrainClient) GetMessage(ctx context.Context, last, capacity in if err != nil { return nil, err } + if c.hook != nil { + callCtx = c.hook.markPoll(callCtx) + } var message *scaleset.RunnerScaleSetMessage err = c.d.effect(callCtx, "observe-poll", nil, func(call context.Context) (Event, error) { var callErr error @@ -331,7 +334,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return ErrQuarantine } - hook := newDrainPollHook("") + hook := newDrainPollHookWithOrigin("", origin) var session Session var sessionID string err = d.effect(ctx, "session-open", nil, func(call context.Context) (Event, error) { @@ -340,6 +343,12 @@ func (d *Driver) drain(ctx context.Context, setID int) error { if err != nil || session == nil { return Event{}, ErrRemote } + hook.mu.Lock() + sessionOrigin := hook.origin + hook.mu.Unlock() + if sessionOrigin != origin { + return Event{}, ErrQuarantine + } current := session.Session() if current.SessionID == [16]byte{} || current.OwnerName != d.Approval.setName() || current.RunnerScaleSet == nil || current.RunnerScaleSet.ID != setID || current.RunnerScaleSet.Name != d.Approval.setName() || current.RunnerScaleSet.RunnerGroupID != d.Approval.RunnerGroupID || !current.RunnerScaleSet.RunnerSetting.DisableUpdate || !slices.ContainsFunc(current.RunnerScaleSet.Labels, func(label scaleset.Label) bool { return label.Name == d.Approval.setName() }) || current.Statistics == nil || current.RunnerScaleSet.Statistics == nil { return Event{}, ErrQuarantine diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 4eea8e8..9b84279 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/actions/scaleset" + "github.com/google/uuid" ) func TestBaselineAcquireTargetIsActionsOnly(t *testing.T) { @@ -265,6 +266,91 @@ func TestBaselineSnapshotRequestsRequireExactOriginBeforeInner(t *testing.T) { } } +func TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner(t *testing.T) { + const target = "https://fixture.invalid/queue?proof=fixture" + for _, tc := range []struct { + name string + mutate func(*http.Request) + }{ + {name: "wrong host", mutate: func(req *http.Request) { req.URL.Host = "other.invalid" }}, + {name: "wrong path", mutate: func(req *http.Request) { req.URL.Path = "/wrong-queue"; req.URL.RawPath = "" }}, + {name: "wrong origin scheme", mutate: func(req *http.Request) { req.URL.Scheme = "http" }}, + {name: "wrong queue proof", mutate: func(req *http.Request) { + query := req.URL.Query() + query.Set("proof", "other") + req.URL.RawQuery = query.Encode() + }}, + } { + t.Run(tc.name, func(t *testing.T) { + hook := newDrainPollHook(target) + hook.mu.Lock() + hook.origin = "https://fixture.invalid:443" + hook.mu.Unlock() + innerCalls := 0 + hook.inner = drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{}`)), Request: req}, nil + }) + session := &drainSyntheticSession{ + client: &http.Client{Transport: sdkRequestMutationRoundTripper{inner: hook, mutate: tc.mutate}}, + initial: scaleset.RunnerScaleSetSession{ + SessionID: uuid.New(), OwnerName: "fixture-owner", MessageQueueURL: target, + Statistics: &scaleset.RunnerScaleSetStatistic{TotalRegisteredRunners: 1, TotalIdleRunners: 1}, + }, + order: new([]string), + } + _, err := runDrainListener(context.Background(), session, 7, hook) + if err == nil { + t.Fatal("marked poll target mismatch was accepted") + } + if innerCalls != 0 { + t.Fatalf("marked poll target mismatch reached inner transport: calls=%d", innerCalls) + } + }) + } +} + +func TestDrainPollHookRejectsApprovalFromDifferentHook(t *testing.T) { + const target = "https://fixture.invalid/queue?proof=fixture" + hook := newDrainPollHook(target) + other := newDrainPollHook(target) + innerCalls := 0 + hook.inner = drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + }) + req, err := http.NewRequestWithContext(other.markPoll(context.Background()), http.MethodGet, target, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, "1") + if _, err := hook.RoundTrip(req); !errors.Is(err, ErrQuarantine) { + t.Fatalf("foreign poll approval error = %v, want quarantine", err) + } + if innerCalls != 0 { + t.Fatalf("foreign poll approval reached inner transport: calls=%d", innerCalls) + } +} + +func TestDrainPollHookForwardsUnmarkedNonPollRequest(t *testing.T) { + hook := newDrainPollHook("https://fixture.invalid/queue?proof=fixture") + innerCalls := 0 + hook.inner = drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + }) + req, err := http.NewRequest(http.MethodPost, "https://fixture.invalid/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", strings.NewReader("{}")) + if err != nil { + t.Fatal(err) + } + if _, err := hook.RoundTrip(req); err != nil { + t.Fatalf("unmarked session request: %v", err) + } + if innerCalls != 1 { + t.Fatalf("unmarked session request inner calls = %d, want one", innerCalls) + } +} + func TestBaselineAcquireTargetRequiresCapturedSessionOrigin(t *testing.T) { for _, tc := range []struct { name string diff --git a/experiments/g01-scaleset/livecanary/drain_test.go b/experiments/g01-scaleset/livecanary/drain_test.go index d1ceca3..e199b74 100644 --- a/experiments/g01-scaleset/livecanary/drain_test.go +++ b/experiments/g01-scaleset/livecanary/drain_test.go @@ -676,6 +676,7 @@ func TestDrainPollHookPreservesStatisticsFieldPresence(t *testing.T) { if err != nil { t.Fatal(err) } + req = req.WithContext(hook.markPoll(req.Context())) req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(drainInitialCapacity)) response, err := hook.RoundTrip(req) if err != nil { @@ -833,6 +834,7 @@ func TestDrainPollHookRejectsDuplicatePhysicalWrites(t *testing.T) { if err != nil { t.Fatal(err) } + req = req.WithContext(hook.markPoll(req.Context())) req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(drainInitialCapacity)) response, err := hook.RoundTrip(req) if err != nil { @@ -881,6 +883,7 @@ func TestDrainPollHookRequiresOneSuccessfulWritePerPoll(t *testing.T) { if err != nil { t.Fatal(err) } + request = request.WithContext(hook.markPoll(request.Context())) request.Header.Set(scaleset.HeaderScaleSetMaxCapacity, strconv.Itoa(drainInitialCapacity)) firstResponse, _ := hook.RoundTrip(request) if firstResponse != nil { diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 45782df..67ce76d 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -400,6 +400,11 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook return nil, ErrQuarantine } hook.mu.Lock() + if hook.origin != "" && hook.origin != wire.requestOrigin() { + hook.invalid = true + hook.mu.Unlock() + return nil, ErrQuarantine + } hook.target = sessionFacts.queueURL hook.origin = wire.requestOrigin() hook.mu.Unlock() diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 718a2c9..f385df7 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -403,6 +403,39 @@ func TestPinnedSDKDrainRejectsSnapshotOriginMismatchBeforeEffects(t *testing.T) } } +func TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + var sessionAttempts atomic.Int32 + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + extraActionsHosts: []string{"other.example.com"}, + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/sessions") && sessionAttempts.Add(1) == 2 { + req.URL.Host = "other.example.com" + req.URL.RawPath = "" + } + }, + }) + j := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("session-open origin mismatch = %v, want quarantine", err) + } + if got := fixture.polls.Load(); got != 0 { + t.Fatalf("session-open origin mismatch started listener polls: polls=%d", got) + } + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { + t.Fatal("session-open origin mismatch produced an observed drain") + } + } +} + func TestPinnedSDKDrainRejectsRunnerSnapshotOriginMismatchBeforeListener(t *testing.T) { a := approval() a.Phases = append(a.Phases, "drain") From a646e4cdefdb59e432cfdafaff1fdd08c1ec4fe4 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 02:11:45 +0900 Subject: [PATCH 34/63] docs(g01): record origin and poll boundary evidence --- docs/evidence/g01-idle-drain.md | 103 ++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 50d406b..a13127d 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -388,6 +388,109 @@ qualification. No credentials, runner/session/JIT operation, workflow operation, app/keychain/launchd/Docker/Lima mutation or cleanup was performed. +### Exact-head follow-up: snapshot-bound session origin and marked listener polls + +Date: 2026-09-13. This correction started from exact head +`8439c12e431bb25bd229c9783119bee125b0c0bd`. The current-head Codex findings +were [session origin not bound to the before snapshot](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996893012) +and [marked listener poll mismatch forwarded before rejection](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996893019). +No Project or Issue ownership/status/goal/dependency field was changed. + +#### Red-first reproductions + +After adding the two behavioral regressions and before changing production +code, this focused command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener|TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner)$' -count=1 -v -timeout=240s +``` + +The session-origin regression observed one listener poll (`polls=1`) instead +of quarantining before listener start. The marked-poll regression observed +the wrong-host, wrong-path and wrong-scheme mutations reach the inner +transport (`calls=1` in each case); the already-existing query validator +rejected the wrong queue-proof case, so that subcase passed in the red run. +The failing output retained only fixed counters and error categories; no +request body, bearer, URL query, response error or private log was recorded. + +#### Corrections and boundary evidence + +The drain driver now initializes the poll hook with the exact origin captured +by the before Scale Set snapshot, and `OpenDrainSession` refuses a different +session-open origin. The driver independently compares `hook.origin` with the +captured origin after session-open and before `runDrainListener`; a mismatch +quarantines without starting a poll. Synthetic API fakes keep the prior +no-wire path, while the pinned SDK path remains origin-bound. + +Listener polls now carry a private per-hook approval marker through the +listener call context. Only a marked poll is admitted to the poll hook's +physical validation: exact queue target/path/query, canonical origin, +capacity and cursor are checked before calling the inner transport, and a +foreign marker or any mismatch is rejected/quarantined before forwarding. +Unmarked session-open, ACK and acquisition requests continue through their +existing baseline wire boundaries. + +The focused green normal and boundary run passed after the corrections: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener|TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner|TestDrainPollHookRejectsApprovalFromDifferentHook|TestDrainPollHookForwardsUnmarkedNonPollRequest)$' -count=1 -v -timeout=240s +``` + +It passed with all four marked-poll mutations rejected before the inner +transport, the foreign marker rejected before the inner transport, and the +unmarked session request forwarded once. The focused race expression covering +the drain, pinned-SDK and baseline boundary families also passed with no race +diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run 'TestDrain|TestPinnedSDKDrain|TestDriverDrainThroughPinnedSDKAndPollHook|TestBaseline.*(Acquire|SessionOpen|Snapshot)' -count=1 -timeout=300s +``` + +The focused normal run passed in 2.358s and the focused race run passed in +4.700s. The full package gates also passed: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary +cd ../.. +bash scripts/gofmt.sh check +git diff --check +set -e +if git diff --text | rg -n '(/Users/|/home/|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})'; then exit 1; fi +``` + +The full normal package passed in 27.179s, the full race package passed in +40.157s, vet was silent and successful, gofmt/diff were clean, and the +secret/private-path scan printed `diff secret/private-path scan passed`. +The repository offline gate was also run from the repository root; it passed +all bounded G01/G02 partitions and printed `offline experiment checks passed: +2 module(s)`. + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +#### Finding matrix, exact head and rollback + +| Finding | Reproduction and resolution | Rollback scope | +|---|---|---| +| [r3996893012](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996893012) session origin | Red regression above reproduced a poll after a mismatched session-open origin; the pinned regression now quarantines with zero listener polls, and the driver/SDK checks bind the origin to the before snapshot. | Revert the correction source/test commit only; no live close, cleanup or rollback operation was run. | +| [r3996893019](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996893019) marked poll forwarding | Red regression above reproduced wrong host/path/scheme forwarding; marked target/path/origin/query and foreign-marker boundaries now reject before inner transport while unmarked non-poll requests retain prior behavior. | Revert the correction source/test commit only; no network or runner rollback was run. | + +The source/test correction is commit +`cbf711cb582ccd6a68eca13312697be21cea580a`; it is the exact implementation +head before this evidence-only follow-up commit and is independently +recoverable with `git revert --no-edit cbf711cb582ccd6a68eca13312697be21cea580a`. +The evidence-only follow-up commit is immediately on that head. The unresolved live gap is the +coordinator-owned exact-head Codex review, required CI and maintainer-authorized +live G01 gate. No live GitHub App, runner, workflow, Docker/Lima, Keychain, +launchd, credential, cleanup or canary operation was performed. + ### Historical exact-head blocker corrections (snapshot `1eb48afb...`; published correction `5d715c486...`) The following entries preserve the historical chronology of the blocker From 1f12e11309b273ca54b2b222f702539425ac91b1 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 02:57:25 +0900 Subject: [PATCH 35/63] fix G01 runtime tenant binding and session cardinality --- .../livecanary/baseline_listener.go | 26 ++- .../livecanary/baseline_terminal.go | 17 +- .../g01-scaleset/livecanary/baseline_wire.go | 210 +++++++++++++++--- experiments/g01-scaleset/livecanary/drain.go | 55 +++-- .../g01-scaleset/livecanary/drain_driver.go | 63 ++++-- .../livecanary/drain_p1_followup_test.go | 71 ++++++ experiments/g01-scaleset/livecanary/sdk.go | 16 +- .../livecanary/sdk_integration_test.go | 67 +++++- 8 files changed, 444 insertions(+), 81 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_listener.go b/experiments/g01-scaleset/livecanary/baseline_listener.go index 46682a4..4aee643 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener.go @@ -33,6 +33,8 @@ type baselineListener struct { session Session initial scaleset.RunnerScaleSetSession sessionID, queue, origin string + runtimePathPrefix string + runtimePathPrefixSet bool running, used, invalid bool after func(context.Context, baselineAcquisition) error } @@ -163,7 +165,7 @@ func (b *baselineListener) finish(r baselineRecord, known bool) (controllerRecor return ref, nil } func (b *baselineListener) wire(stage string) *baselineWireCapture { - return &baselineWireCapture{stage: stage, setID: b.setID, organization: b.approval.Organization, owner: b.approval.setName(), queue: b.queue, origin: b.origin, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} + return &baselineWireCapture{stage: stage, setID: b.setID, organization: b.approval.Organization, owner: b.approval.setName(), queue: b.queue, origin: b.origin, runtimePathPrefix: b.runtimePathPrefix, runtimePathPrefixSet: b.runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} } func (b *baselineListener) capturedOrigin() string { @@ -174,6 +176,15 @@ func (b *baselineListener) capturedOrigin() string { defer b.mu.Unlock() return b.origin } + +func (b *baselineListener) capturedRuntimePathPrefix() (string, bool) { + if b == nil { + return "", false + } + b.mu.Lock() + defer b.mu.Unlock() + return b.runtimePathPrefix, b.runtimePathPrefixSet && !b.invalid +} func (b *baselineListener) run(after func(context.Context, baselineAcquisition) error) error { if b == nil || b.ctx == nil || b.cancel == nil || !b.mu.TryLock() { return ErrQuarantine @@ -234,7 +245,12 @@ func (b *baselineListener) initialize() error { cancel() r.Set = c.set r.HTTPStatus = c.status - known := c.observed() && r.Set.eligible(b.approval, b.setID) && ((callErr != nil && b.ctx.Err() != nil) || (callErr == nil && set != nil && set.ID == r.Set.ID && set.Name == r.Set.Name && set.RunnerGroupID == r.Set.GroupID && set.RunnerSetting.DisableUpdate && r.Set.Statistics.matches(set.Statistics))) + prefix, prefixKnown := c.requestRuntimePathPrefix() + known := c.observed() && prefixKnown && r.Set.eligible(b.approval, b.setID) && ((callErr != nil && b.ctx.Err() != nil) || (callErr == nil && set != nil && set.ID == r.Set.ID && set.Name == r.Set.Name && set.RunnerGroupID == r.Set.GroupID && set.RunnerSetting.DisableUpdate && r.Set.Statistics.matches(set.Statistics))) + if prefixKnown { + b.runtimePathPrefix = prefix + b.runtimePathPrefixSet = true + } if _, err = b.finish(r, known); err != nil { return err } @@ -248,9 +264,10 @@ func (b *baselineListener) initialize() error { cancel() sf, _, _, status := c.facts() origin := c.requestOrigin() + sessionPrefix, sessionPrefixKnown := c.requestRuntimePathPrefix() r.Session = sf r.HTTPStatus = status - known = c.observed() && sf.eligible(b.approval, b.setID) && ((callErr != nil && b.ctx.Err() != nil) || (callErr == nil && session != nil)) + known = c.observed() && sessionPrefixKnown && sf.eligible(b.approval, b.setID) && ((callErr != nil && b.ctx.Err() != nil) || (callErr == nil && session != nil)) var initial scaleset.RunnerScaleSetSession if callErr == nil && session != nil { initial = session.Session() @@ -258,6 +275,9 @@ func (b *baselineListener) initialize() error { if known && callErr == nil { known = initial.SessionID.String() == sf.SessionID && initial.OwnerName == sf.Owner && sf.Statistics.matches(initial.Statistics) && initial.MessageQueueURL != "" } + if known && (origin == "" || !b.runtimePathPrefixSet || sessionPrefix != b.runtimePathPrefix) { + known = false + } if known && origin != "" { b.origin = origin } diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal.go b/experiments/g01-scaleset/livecanary/baseline_terminal.go index 7d2aacc..018d349 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal.go @@ -131,7 +131,11 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { r.Terminal.Roster = &observation known = e == nil && state.rosterMatches(&observation, func(ref controllerRecordRef) *Event { return s.event(workerRef(ref)) }) case stage == "terminal-set" || stage == "terminal-set-recheck" || stage == "terminal-set-absence": - capture := &baselineWireCapture{stage: stage, setID: s.setID, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} + runtimePathPrefix, runtimePathPrefixSet := s.listener.capturedRuntimePathPrefix() + if !runtimePathPrefixSet { + return ErrQuarantine + } + capture := &baselineWireCapture{stage: stage, setID: s.setID, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) set, e := s.captured.GetScaleSet(capture.context(ctx), s.setID) callErr = e @@ -152,10 +156,11 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { return ErrQuarantine } origin := s.listener.capturedOrigin() - if origin == "" { + runtimePathPrefix, runtimePathPrefixSet := s.listener.capturedRuntimePathPrefix() + if origin == "" || !runtimePathPrefixSet { return ErrQuarantine } - capture := &baselineWireCapture{stage: stage, setID: s.setID, sessionID: state.sessionID, origin: origin} + capture := &baselineWireCapture{stage: stage, setID: s.setID, sessionID: state.sessionID, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) callErr = s.listener.session.Close(capture.context(ctx)) cancel() @@ -171,7 +176,11 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { } known = e == nil && r.Terminal.Deletion != nil && receipt.AbsenceResult != nil case stage == "terminal-set-delete": - capture := &baselineWireCapture{stage: stage, setID: s.setID, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} + runtimePathPrefix, runtimePathPrefixSet := s.listener.capturedRuntimePathPrefix() + if !runtimePathPrefixSet { + return ErrQuarantine + } + capture := &baselineWireCapture{stage: stage, setID: s.setID, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) callErr = s.captured.DeleteScaleSet(capture.context(ctx), s.setID) cancel() diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index dbbedfa..5b51164 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -18,32 +18,117 @@ import ( type baselineWireKey struct{} type baselineWireCapture struct { - mu sync.Mutex - stage string - setID int - organization string - owner string - runnerName string - sessionID string - queue string // private, captured from the exact session; never journaled - allowedHosts []string - cursor int - count, status int - requestIDs []int64 - requestCount int - origin string - invalid bool - session *baselineSessionFacts - batch *baselineBatch - accepted *baselineAccepted - set *baselineSetFacts - runner *baselineRunnerFacts - jit *scaleset.RunnerScaleSetJitRunnerConfig + mu sync.Mutex + stage string + setID int + organization string + owner string + runnerName string + sessionID string + queue string // private, captured from the exact session; never journaled + runtimePathPrefix string // private tenant path prefix; never journaled + runtimePathPrefixSet bool + allowedHosts []string + cursor int + count, status int + requestIDs []int64 + requestCount int + origin string + invalid bool + session *baselineSessionFacts + batch *baselineBatch + accepted *baselineAccepted + set *baselineSetFacts + runner *baselineRunnerFacts + jit *scaleset.RunnerScaleSetJitRunnerConfig } func (c *baselineWireCapture) context(ctx context.Context) context.Context { return context.WithValue(ctx, baselineWireKey{}, c) } + +// baselinePathPrefix returns the path before an exact endpoint marker. The +// prefix is intentionally retained only in memory: it binds later marked +// requests to the first approved tenant/runtime route without putting private +// URLs in journal evidence. +func baselinePathPrefix(path, marker string) (string, bool) { + if path == "" || marker == "" || !strings.HasSuffix(path, marker) { + return "", false + } + prefix := strings.TrimSuffix(path, marker) + if prefix != "" && (!strings.HasPrefix(prefix, "/") || strings.HasSuffix(prefix, "/")) { + return "", false + } + if prefix == "" { + prefix = "/" + } + return prefix, true +} + +func baselineRuntimeScaleSetPrefix(path string, setID int) (string, bool) { + return baselineRuntimeScaleSetPrefixForTail(path, setID, "") +} + +func baselineRuntimeScaleSetPrefixForTail(path string, setID int, tail string) (string, bool) { + if setID <= 0 { + return "", false + } + return baselinePathPrefix(path, "/_apis/runtime/runnerscalesets/"+strconv.Itoa(setID)+tail) +} + +func baselineRuntimeRunnerPrefix(path string) (string, bool) { + return baselinePathPrefix(path, "/_apis/distributedtask/pools/0/agents") +} + +func baselineRuntimeScaleSetPath(prefix string, setID int, tail string) string { + if prefix == "/" { + prefix = "" + } + return prefix + "/_apis/runtime/runnerscalesets/" + strconv.Itoa(setID) + tail +} + +func baselineRuntimeRunnerPath(prefix string) string { + if prefix == "/" { + prefix = "" + } + return prefix + "/_apis/distributedtask/pools/0/agents" +} + +func (c *baselineWireCapture) runtimePrefixMatches(prefix string) bool { + return c == nil || !c.runtimePathPrefixSet || c.runtimePathPrefix == prefix +} + +func (c *baselineWireCapture) runtimeRequestPrefix(r *http.Request) (string, bool) { + if c == nil || r == nil || r.URL == nil { + return "", false + } + var ( + prefix string + ok bool + ) + if c.stage == "runner-observe" { + prefix, ok = baselineRuntimeRunnerPrefix(r.URL.Path) + } else { + tail := "" + switch c.stage { + case "session-open": + tail = "/sessions" + case "acquire": + tail = "/acquirejobs" + case "terminal-session-close": + if c.sessionID == "" { + return "", false + } + tail = "/sessions/" + c.sessionID + } + prefix, ok = baselineRuntimeScaleSetPrefixForTail(r.URL.Path, c.setID, tail) + } + if !ok || !c.runtimePrefixMatches(prefix) { + return "", false + } + return prefix, true +} + func (c *baselineWireCapture) target(r *http.Request) bool { if r == nil || r.URL == nil || r.URL.Fragment != "" || r.URL.User != nil || r.URL.EscapedPath() != r.URL.Path { return false @@ -73,25 +158,25 @@ func (c *baselineWireCapture) target(r *http.Request) bool { if c.stage == "terminal-set-delete" { method = "DELETE" } - return r.Method == method && strings.HasSuffix(r.URL.Path, strings.TrimSuffix(suffix, "/")) && baselineExactAPIVersionQuery(r.URL.RawQuery) + return r.Method == method && c.runtimeRequestTarget(r, "") && baselineExactAPIVersionQuery(r.URL.RawQuery) } if c.stage == "terminal-session-close" { origin, ok := baselineRequestOrigin(r.URL) - return c.origin != "" && ok && origin == c.origin && c.sessionID != "" && r.Method == "DELETE" && strings.HasSuffix(r.URL.Path, suffix+"sessions/"+c.sessionID) && baselineExactAPIVersionQuery(r.URL.RawQuery) + return c.origin != "" && ok && origin == c.origin && c.sessionID != "" && r.Method == "DELETE" && c.runtimeRequestTarget(r, "sessions/"+c.sessionID) && baselineExactAPIVersionQuery(r.URL.RawQuery) } if c.stage == "session-open" { if len(c.allowedHosts) > 0 && !baselineOriginAllowed(r.URL, c.allowedHosts) { return false } - suffix += "sessions" + return r.Method == "POST" && c.runtimeRequestTarget(r, "sessions") && baselineExactAPIVersionQuery(r.URL.RawQuery) } else if c.stage == "runner-observe" { - return c.snapshotOriginAllowed(r) && r.Method == http.MethodGet && strings.HasSuffix(r.URL.Path, "/_apis/distributedtask/pools/0/agents") && baselineExactQuery(r.URL.RawQuery, map[string]string{"agentName": c.runnerName, "api-version": "6.0-preview"}) + return c.snapshotOriginAllowed(r) && r.Method == http.MethodGet && c.runtimeRequestTarget(r, "") && baselineExactQuery(r.URL.RawQuery, map[string]string{"agentName": c.runnerName, "api-version": "6.0-preview"}) } else if c.stage == "acquire" { origin, ok := baselineRequestOrigin(r.URL) if c.origin == "" || !ok || origin != c.origin || len(c.allowedHosts) == 0 || !baselineOriginAllowed(r.URL, c.allowedHosts) { return false } - suffix = "/_apis/runtime" + suffix + "acquirejobs" + return r.Method == "POST" && c.runtimeRequestTarget(r, "acquirejobs") && baselineExactAPIVersionQuery(r.URL.RawQuery) } else if c.stage == "jit" { suffix += "generatejitconfig" } else { @@ -100,6 +185,25 @@ func (c *baselineWireCapture) target(r *http.Request) bool { return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && baselineExactAPIVersionQuery(r.URL.RawQuery) } +func (c *baselineWireCapture) runtimeRequestTarget(r *http.Request, tail string) bool { + if c == nil || r == nil || r.URL == nil { + return false + } + if c.stage == "runner-observe" { + prefix, ok := baselineRuntimeRunnerPrefix(r.URL.Path) + return ok && c.runtimePrefixMatches(prefix) && r.URL.Path == baselineRuntimeRunnerPath(prefix) + } + prefixTail := "" + if tail != "" { + prefixTail = "/" + tail + } + prefix, ok := baselineRuntimeScaleSetPrefixForTail(r.URL.Path, c.setID, prefixTail) + if !ok || !c.runtimePrefixMatches(prefix) { + return false + } + return r.URL.Path == baselineRuntimeScaleSetPath(prefix, c.setID, "/"+tail) || (tail == "" && r.URL.Path == baselineRuntimeScaleSetPath(prefix, c.setID, "")) +} + func (c *baselineWireCapture) snapshotOriginAllowed(r *http.Request) bool { if c == nil || r == nil || r.URL == nil || len(c.allowedHosts) == 0 || !baselineOriginAllowed(r.URL, c.allowedHosts) { return false @@ -203,7 +307,7 @@ func baselineWireAllowedHosts(a Approval, apiHost string) []string { // baselineRequestCaptureTransport runs immediately above the physical // transport. User-supplied test wrappers may mutate a request before it gets // here, so this is the final request-side boundary before any bytes leave the -// process. Only explicitly marked session-open and acquisition requests are inspected. +// process. Only explicitly marked G01 runtime requests are inspected. type baselineRequestCaptureTransport struct{ inner http.RoundTripper } func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) { @@ -257,32 +361,53 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { } return c.rejectRequest(req) } + c.mu.Lock() + if c.requestCount != 0 || c.invalid { + c.mu.Unlock() + return c.rejectRequest(req) + } + c.requestCount = 1 + c.mu.Unlock() data, err := readBaselineRequestBody(req.Body) req.Body = nil if err != nil || !validBaselineSessionOpenBody(data, c.owner) { clear(data) - return c.rejectRequest(req) + return c.rejectReservedRequest(req) } installBaselineRequestBody(req, data) origin, ok := baselineRequestOrigin(req.URL) if !ok { - return c.rejectRequest(req) + return c.rejectReservedRequest(req) + } + prefix, ok := c.runtimeRequestPrefix(req) + if !ok { + return c.rejectReservedRequest(req) } c.mu.Lock() if c.origin != "" && c.origin != origin { c.invalid = true } c.origin = origin + if !c.runtimePathPrefixSet { + c.runtimePathPrefix = prefix + c.runtimePathPrefixSet = true + } else if c.runtimePathPrefix != prefix { + c.invalid = true + } c.mu.Unlock() return nil } - if c.stage == "set-observe" || c.stage == "runner-observe" { + if c.stage == "set-observe" || c.stage == "runner-observe" || strings.HasPrefix(c.stage, "terminal-set") || c.stage == "terminal-session-close" { if !snapshotRequestCandidate(req) { return nil } if !c.target(req) { return c.rejectRequest(req) } + prefix, ok := c.runtimeRequestPrefix(req) + if !ok { + return c.rejectRequest(req) + } origin, ok := baselineRequestOrigin(req.URL) if !ok { return c.rejectRequest(req) @@ -294,6 +419,12 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { } else if c.origin != origin { c.invalid = true } + if !c.runtimePathPrefixSet { + c.runtimePathPrefix = prefix + c.runtimePathPrefixSet = true + } else if c.runtimePathPrefix != prefix { + c.invalid = true + } valid := c.requestCount == 1 && !c.invalid c.mu.Unlock() if !valid { @@ -375,6 +506,16 @@ func (c *baselineWireCapture) rejectRequest(req *http.Request) error { return ErrRemote } +func (c *baselineWireCapture) rejectReservedRequest(req *http.Request) error { + c.mu.Lock() + c.invalid = true + c.mu.Unlock() + if req != nil && req.Body != nil { + _ = req.Body.Close() + } + return ErrRemote +} + type baselineRequestBody struct { mu sync.Mutex reader *bytes.Reader @@ -404,6 +545,15 @@ func (c *baselineWireCapture) requestOrigin() string { return c.origin } +func (c *baselineWireCapture) requestRuntimePathPrefix() (string, bool) { + if c == nil { + return "", false + } + c.mu.Lock() + defer c.mu.Unlock() + return c.runtimePathPrefix, c.runtimePathPrefixSet && !c.invalid +} + func (c *baselineWireCapture) requestObserved() bool { if c == nil { return false diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index c83653f..5e3f906 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -332,25 +332,27 @@ type drainPollHook struct { response chan struct{} release chan struct{} - mu sync.Mutex - wroteRequest bool - withdrawalCompleted bool - responseBeforeWrite bool - responseHeld bool - invalid bool - pollAttempts int - statistics [3]drainStatistics - statisticsKnown [3]bool - batches [3]*baselineBatch - batchesKnown [3]bool - messagePresent [3]bool - messageAbsent [3]bool - wroteCallbacks [3]int - onRequestWritten func() - wroteOnce sync.Once - withdrawalOnce sync.Once - responseOnce sync.Once - releaseOnce sync.Once + mu sync.Mutex + runtimePathPrefix string + runtimePathPrefixSet bool + wroteRequest bool + withdrawalCompleted bool + responseBeforeWrite bool + responseHeld bool + invalid bool + pollAttempts int + statistics [3]drainStatistics + statisticsKnown [3]bool + batches [3]*baselineBatch + batchesKnown [3]bool + messagePresent [3]bool + messageAbsent [3]bool + wroteCallbacks [3]int + onRequestWritten func() + wroteOnce sync.Once + withdrawalOnce sync.Once + responseOnce sync.Once + releaseOnce sync.Once } func newDrainPollHook(target string) *drainPollHook { @@ -362,9 +364,17 @@ func newDrainPollHook(target string) *drainPollHook { } func newDrainPollHookWithOrigin(target, origin string) *drainPollHook { + return newDrainPollHookWithOriginAndPrefix(target, origin, "") +} + +func newDrainPollHookWithOriginAndPrefix(target, origin, runtimePathPrefix string) *drainPollHook { hook := newDrainPollHook(target) hook.mu.Lock() hook.origin = origin + if runtimePathPrefix != "" { + hook.runtimePathPrefix = runtimePathPrefix + hook.runtimePathPrefixSet = true + } hook.mu.Unlock() return hook } @@ -469,10 +479,17 @@ func (h *drainPollHook) pollRequestValid(req *http.Request, attempt int) bool { } h.mu.Lock() approvedOrigin := h.origin + runtimePathPrefixSet := h.runtimePathPrefixSet h.mu.Unlock() if (approvedOrigin == "" && strings.EqualFold(want.Scheme, "https")) || (approvedOrigin != "" && requestOrigin != approvedOrigin) { return false } + // Production queue URLs are HTTPS and must only be marked after the + // scale-set snapshot has captured a runtime tenant prefix. HTTP remains + // available for the in-process synthetic listener seams. + if strings.EqualFold(want.Scheme, "https") && !runtimePathPrefixSet { + return false + } values := req.Header.Values(scaleset.HeaderScaleSetMaxCapacity) wantCapacity := drainInitialCapacity if attempt == 2 { diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index e4863b1..0864a0c 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -132,8 +132,13 @@ func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error if c.hook != nil { c.hook.mu.Lock() queue := c.hook.target + runtimePathPrefix := c.hook.runtimePathPrefix + runtimePathPrefixSet := c.hook.runtimePathPrefixSet c.hook.mu.Unlock() - wire = &baselineWireCapture{stage: "ack", queue: queue, cursor: id} + if !runtimePathPrefixSet { + return Event{}, c.reject("ack") + } + wire = &baselineWireCapture{stage: "ack", queue: queue, cursor: id, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet} call = wire.context(call) } if err := c.inner.DeleteMessage(call, id); err != nil { @@ -168,12 +173,17 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] } c.hook.mu.Lock() queue, origin := c.hook.target, c.hook.origin + runtimePathPrefix := c.hook.runtimePathPrefix + runtimePathPrefixSet := c.hook.runtimePathPrefixSet c.hook.mu.Unlock() + if !runtimePathPrefixSet { + return Event{}, c.reject("acquire") + } apiHost := "" if reader, ok := c.d.API.(drainEndpointHostReader); ok { apiHost = reader.drainEndpointHost() } - wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, requestIDs: slices.Clone(ids), origin: origin, allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} + wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, requestIDs: slices.Clone(ids), origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} call = wire.context(call) } got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) @@ -203,14 +213,24 @@ func (d *Driver) drainSnapshot(ctx context.Context, setID int, stage string) (dr } func (d *Driver) drainSnapshotWithOrigin(ctx context.Context, setID int, stage, expectedOrigin string) (drainSnapshot, string, error) { + snapshot, origin, _, err := d.drainSnapshotWithBinding(ctx, setID, stage, expectedOrigin, "") + return snapshot, origin, err +} + +// drainSnapshotWithBinding captures the first approved runtime tenant prefix +// from the scale-set request and requires the runner request to use that same +// prefix. A non-empty expected prefix binds a later snapshot to the prefix +// already established by the drain phase. +func (d *Driver) drainSnapshotWithBinding(ctx context.Context, setID int, stage, expectedOrigin, expectedRuntimePathPrefix string) (drainSnapshot, string, string, error) { var snapshot drainSnapshot var set *scaleset.RunnerScaleSet setReader, setWire := d.API.(drainScaleSetWireReader) runnerReader, runnerWire := d.API.(drainRunnerWireReader) if setWire != runnerWire || (expectedOrigin != "" && !setWire) { - return drainSnapshot{}, "", ErrQuarantine + return drainSnapshot{}, "", "", ErrQuarantine } origin := expectedOrigin + runtimePathPrefix := expectedRuntimePathPrefix allowedHosts := []string(nil) if reader, ok := d.API.(drainEndpointHostReader); ok { allowedHosts = baselineWireAllowedHosts(d.Approval, reader.drainEndpointHost()) @@ -218,14 +238,16 @@ func (d *Driver) drainSnapshotWithOrigin(ctx context.Context, setID int, stage, if err := d.effect(ctx, "observe-owned", nil, func(call context.Context) (Event, error) { var err error if setWire { - wire := &baselineWireCapture{stage: "set-observe", setID: setID, origin: expectedOrigin, allowedHosts: allowedHosts} + wire := &baselineWireCapture{stage: "set-observe", setID: setID, origin: expectedOrigin, runtimePathPrefix: expectedRuntimePathPrefix, runtimePathPrefixSet: expectedRuntimePathPrefix != "", allowedHosts: allowedHosts} set, err = setReader.drainGetScaleSet(call, setID, wire) wireSet, status := wire.setFacts() wireOrigin := wire.requestOrigin() - if err != nil || !wire.observed() || status != 200 || wireOrigin == "" || (origin != "" && wireOrigin != origin) || !wireSet.eligibleForDrain(d.Approval, setID) || !wireSet.matches(set) { + wireRuntimePathPrefix, prefixKnown := wire.requestRuntimePathPrefix() + if err != nil || !wire.observed() || status != 200 || wireOrigin == "" || !prefixKnown || (origin != "" && wireOrigin != origin) || !wireSet.eligibleForDrain(d.Approval, setID) || !wireSet.matches(set) { return Event{}, ErrQuarantine } origin = wireOrigin + runtimePathPrefix = wireRuntimePathPrefix } else { // Synthetic API fakes used by state-machine tests have no physical // request origin. Production SDKAPI always takes the wire branch. @@ -249,17 +271,18 @@ func (d *Driver) drainSnapshotWithOrigin(ctx context.Context, setID int, stage, snapshot.StatsKnown = true return Event{ID: set.ID}, nil }); err != nil { - return drainSnapshot{}, "", err + return drainSnapshot{}, "", "", err } var runner *scaleset.RunnerReference if err := d.effect(ctx, "observe-runner", nil, func(call context.Context) (Event, error) { var err error if runnerWire { - wire := &baselineWireCapture{stage: "runner-observe", runnerName: d.Approval.workerName(), origin: origin, allowedHosts: allowedHosts} + wire := &baselineWireCapture{stage: "runner-observe", runnerName: d.Approval.workerName(), origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefix != "", allowedHosts: allowedHosts} runner, err = runnerReader.drainFindRunner(call, d.Approval.workerName(), wire) wireRunner, status := wire.runnerFacts() wireOrigin := wire.requestOrigin() - if err != nil || !wire.observed() || status != http.StatusOK || origin == "" || wireOrigin != origin || !wireRunner.matches(runner) { + wireRuntimePathPrefix, prefixKnown := wire.requestRuntimePathPrefix() + if err != nil || !wire.observed() || status != http.StatusOK || origin == "" || wireOrigin != origin || !prefixKnown || wireRuntimePathPrefix != runtimePathPrefix || !wireRunner.matches(runner) { return Event{}, ErrQuarantine } } else { @@ -279,10 +302,10 @@ func (d *Driver) drainSnapshotWithOrigin(ctx context.Context, setID int, stage, snapshot.Runner = &drainRunnerIdentity{ID: runner.ID, Name: runner.Name, ScaleSetID: runner.RunnerScaleSetID} return Event{ID: runner.ID, DrainSnapshot: &snapshot, DrainSnapshotStage: stage}, nil }); err != nil { - return drainSnapshot{}, "", err + return drainSnapshot{}, "", "", err } _ = stage // Stage is retained by the caller's before/after final payload. - return snapshot, origin, nil + return snapshot, origin, runtimePathPrefix, nil } func validDrainIdlePrerequisite(s drainSnapshot) bool { @@ -321,7 +344,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { return ErrQuarantine } phaseSequence := phaseState.drainPhaseSequence - before, origin, err := d.drainSnapshotWithOrigin(ctx, setID, "before", "") + before, origin, runtimePathPrefix, err := d.drainSnapshotWithBinding(ctx, setID, "before", "", "") if err != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, err)); markerErr != nil { return markerErr @@ -334,7 +357,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return ErrQuarantine } - hook := newDrainPollHookWithOrigin("", origin) + hook := newDrainPollHookWithOriginAndPrefix("", origin, runtimePathPrefix) var session Session var sessionID string err = d.effect(ctx, "session-open", nil, func(call context.Context) (Event, error) { @@ -345,8 +368,10 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } hook.mu.Lock() sessionOrigin := hook.origin + sessionRuntimePathPrefix := hook.runtimePathPrefix + sessionRuntimePathPrefixSet := hook.runtimePathPrefixSet hook.mu.Unlock() - if sessionOrigin != origin { + if sessionOrigin != origin || (runtimePathPrefix != "" && (!sessionRuntimePathPrefixSet || sessionRuntimePathPrefix != runtimePathPrefix)) { return Event{}, ErrQuarantine } current := session.Session() @@ -388,7 +413,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, runErr)); markerErr != nil { return markerErr } - after, _, afterErr := d.drainSnapshotWithOrigin(ctx, setID, "after", origin) + after, _, _, afterErr := d.drainSnapshotWithBinding(ctx, setID, "after", origin, runtimePathPrefix) if afterErr != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, afterErr)); markerErr != nil { return markerErr @@ -404,15 +429,19 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } closeErr := d.effect(ctx, "session-close", nil, func(call context.Context) (Event, error) { origin := "" + runtimePathPrefix := "" + runtimePathPrefixSet := false if hook != nil { hook.mu.Lock() origin = hook.origin + runtimePathPrefix = hook.runtimePathPrefix + runtimePathPrefixSet = hook.runtimePathPrefixSet hook.mu.Unlock() } - if origin == "" { + if origin == "" || !runtimePathPrefixSet { return Event{}, ErrQuarantine } - wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID, origin: origin} + wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet} call = wire.context(call) if err := session.Close(call); err != nil { return Event{}, err @@ -429,7 +458,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } return closeErr } - after, _, afterErr := d.drainSnapshotWithOrigin(ctx, setID, "after", origin) + after, _, _, afterErr := d.drainSnapshotWithBinding(ctx, setID, "after", origin, runtimePathPrefix) if afterErr != nil { if markerErr := d.recordDrainMarker(drainMarkerFor(ctx, afterErr)); markerErr != nil { return markerErr diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 9b84279..b813091 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -229,6 +229,37 @@ func TestBaselineSessionOpenBodyMustMatchOwnerBeforeInner(t *testing.T) { } } +func TestBaselineSessionOpenRejectsDuplicateMarkedPOSTBeforeInner(t *testing.T) { + capture := &baselineWireCapture{ + stage: "session-open", + setID: 7, + owner: "g01-test", + organization: "fixture-org", + allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + request := func() *http.Request { + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", strings.NewReader(`{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"g01-test"}`)) + if err != nil { + t.Fatal(err) + } + return req + } + if _, err := transport.RoundTrip(request()); err != nil { + t.Fatalf("first marked session-open = %v, want forwarding", err) + } + if _, err := transport.RoundTrip(request()); !errors.Is(err, ErrRemote) { + t.Fatalf("duplicate marked session-open = %v, want remote rejection", err) + } + if innerCalls != 1 { + t.Fatalf("duplicate marked session-open reached inner transport: calls=%d, want one", innerCalls) + } +} + func TestBaselineSnapshotRequestsRequireExactOriginBeforeInner(t *testing.T) { for _, tc := range []struct { name string @@ -266,6 +297,46 @@ func TestBaselineSnapshotRequestsRequireExactOriginBeforeInner(t *testing.T) { } } +func TestBaselineRuntimePathPrefixMismatchStopsBeforeInner(t *testing.T) { + const ( + origin = "https://api.example:443" + prefix = "/tenant/approved" + ) + tests := []struct { + name string + stage string + method string + path string + body string + }{ + {name: "scale-set snapshot", stage: "set-observe", method: http.MethodGet, path: "/tenant/foreign/_apis/runtime/runnerscalesets/7?api-version=6.0-preview"}, + {name: "runner snapshot", stage: "runner-observe", method: http.MethodGet, path: "/tenant/foreign/_apis/distributedtask/pools/0/agents?agentName=g01-test-worker-1&api-version=6.0-preview"}, + {name: "session open", stage: "session-open", method: http.MethodPost, path: "/tenant/foreign/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", body: `{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"g01-test"}`}, + {name: "acquisition", stage: "acquire", method: http.MethodPost, path: "/tenant/foreign/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", body: `[41]`}, + {name: "session close", stage: "terminal-session-close", method: http.MethodDelete, path: "/tenant/foreign/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{stage: tc.stage, setID: 7, owner: "g01-test", runnerName: "g01-test-worker-1", sessionID: "session", origin: origin, runtimePathPrefix: prefix, runtimePathPrefixSet: true, requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, "https://api.example"+tc.path, strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("same-origin foreign-prefix request = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("same-origin foreign-prefix request reached inner transport: calls=%d", innerCalls) + } + }) + } +} + func TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner(t *testing.T) { const target = "https://fixture.invalid/queue?proof=fixture" for _, tc := range []struct { diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 67ce76d..fcece25 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -390,23 +390,31 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook if configured == nil || configured.client == nil { return nil, ErrRemote } - wire := &baselineWireCapture{stage: "session-open", setID: id, organization: configured.approval.Organization, owner: owner, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} + hook.mu.Lock() + expectedRuntimePathPrefix := hook.runtimePathPrefix + expectedRuntimePathPrefixSet := hook.runtimePathPrefixSet + hook.mu.Unlock() + wire := &baselineWireCapture{stage: "session-open", setID: id, organization: configured.approval.Organization, owner: owner, runtimePathPrefix: expectedRuntimePathPrefix, runtimePathPrefixSet: expectedRuntimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} session, err := configured.client.MessageSessionClient(wire.context(c), id, owner, configured.options...) if err != nil { return nil, ErrRemote } sessionFacts, _, _, status := wire.facts() - if session == nil || !wire.observed() || status != http.StatusOK || wire.requestOrigin() == "" || !validDrainSessionWire(configured.approval, id, owner, sessionFacts, session.Session()) { + sessionOrigin := wire.requestOrigin() + sessionRuntimePathPrefix, prefixKnown := wire.requestRuntimePathPrefix() + if session == nil || !wire.observed() || status != http.StatusOK || sessionOrigin == "" || !prefixKnown || !validDrainSessionWire(configured.approval, id, owner, sessionFacts, session.Session()) { return nil, ErrQuarantine } hook.mu.Lock() - if hook.origin != "" && hook.origin != wire.requestOrigin() { + if (hook.origin != "" && hook.origin != sessionOrigin) || (hook.runtimePathPrefixSet && hook.runtimePathPrefix != sessionRuntimePathPrefix) { hook.invalid = true hook.mu.Unlock() return nil, ErrQuarantine } hook.target = sessionFacts.queueURL - hook.origin = wire.requestOrigin() + hook.origin = sessionOrigin + hook.runtimePathPrefix = sessionRuntimePathPrefix + hook.runtimePathPrefixSet = true hook.mu.Unlock() return session, nil } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index f385df7..0ca50ec 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -403,6 +403,34 @@ func TestPinnedSDKDrainRejectsSnapshotOriginMismatchBeforeEffects(t *testing.T) } } +func TestPinnedSDKDrainRejectsSnapshotTenantPrefixMismatchBeforeEffects(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + var snapshotRequests atomic.Int32 + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + runtimeActionsBasePath: "/tenant/approved/", + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodGet && strings.HasSuffix(req.URL.Path, "/runnerscalesets/7") && snapshotRequests.Add(1) == 2 { + req.URL.Path = "/tenant/foreign/_apis/runtime/runnerscalesets/7" + req.URL.RawPath = "" + } + }, + }) + j := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("snapshot tenant-prefix mismatch = %v, want quarantine", err) + } + if got := fixture.snapshotReads.Load(); got != 1 { + t.Fatalf("mismatched after snapshot reached fixture: reads=%d, want before only", got) + } +} + func TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener(t *testing.T) { a := approval() a.Phases = append(a.Phases, "drain") @@ -436,6 +464,37 @@ func TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener(t *testing.T) } } +func TestPinnedSDKDrainRejectsSessionTenantPrefixMismatchBeforeListener(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + runtimeActionsBasePath: "/tenant/approved/", + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/sessions") { + req.URL.Path = "/tenant/foreign/_apis/runtime/runnerscalesets/7/sessions" + req.URL.RawPath = "" + } + }, + }) + fixture.sessionOpens.Store(0) + j := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { + t.Fatalf("session tenant-prefix mismatch = %v, want quarantine", err) + } + if got := fixture.sessionOpens.Load(); got != 0 { + t.Fatalf("mismatched session-open reached fixture: opens=%d", got) + } + if got := fixture.polls.Load(); got != 0 { + t.Fatalf("session tenant-prefix mismatch started listener polls: polls=%d", got) + } +} + func TestPinnedSDKDrainRejectsRunnerSnapshotOriginMismatchBeforeListener(t *testing.T) { a := approval() a.Phases = append(a.Phases, "drain") @@ -783,8 +842,8 @@ func TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete(t *testing.T) { if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { t.Fatalf("wrong session-close target = %v, want quarantine", err) } - if fixture.closeRequests.Load() != 1 { - t.Fatalf("wrong session-close target requests = %d, want one loopback attempt", fixture.closeRequests.Load()) + if fixture.closeRequests.Load() != 0 { + t.Fatalf("wrong session-close target requests = %d, want pre-inner rejection", fixture.closeRequests.Load()) } for _, event := range journal.Events() { if event.Kind == "result" && event.Operation == "session-close" { @@ -816,8 +875,8 @@ func TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin(t *testing.T) { if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrQuarantine) { t.Fatalf("wrong session-close origin = %v, want quarantine", err) } - if fixture.closeRequests.Load() != 1 { - t.Fatalf("wrong session-close origin requests = %d, want one loopback attempt", fixture.closeRequests.Load()) + if fixture.closeRequests.Load() != 0 { + t.Fatalf("wrong session-close origin requests = %d, want pre-inner rejection", fixture.closeRequests.Load()) } for _, event := range j.Events() { if event.Kind == "result" && event.Operation == "session-close" { From 1bb0c878fe97b81cae61f3416cd074670ec2292d Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 02:59:01 +0900 Subject: [PATCH 36/63] docs: record G01 exact-head follow-up evidence --- docs/evidence/g01-idle-drain.md | 84 +++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index a13127d..6f78ee1 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1528,3 +1528,87 @@ workflow, runner, credential, App, Keychain, launchd, Docker, Lima, live cleanup, or rollback operation was performed. Independent final review, exact-head Codex review, CI, and the live G01 evidence gate remain coordinator-owned, and the parent G01 goal remains active. + +### Exact-head follow-up: runtime tenant-prefix binding and session-open cardinality + +Date: 2026-09-12 UTC. This correction addresses the fresh exact-head Codex +P1 findings [runtime path-prefix binding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996992590) +and [duplicate marked session-open cardinality](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996992592). + +The meaningful red-first command was run before the implementation change: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineSessionOpenRejectsDuplicateMarkedPOSTBeforeInner|TestPinnedSDKDrainRejectsSnapshotTenantPrefixMismatchBeforeEffects|TestPinnedSDKDrainRejectsSessionTenantPrefixMismatchBeforeListener)$' -count=1 -v -timeout=180s +``` + +It exited 1. The duplicate marked session-open returned nil on its second +request instead of `ErrRemote`; the same-origin foreign-prefix after-snapshot +and session-open cases returned nil instead of quarantine. The failing cases +also showed the pre-fix request could reach the inner/fixture boundary, while +the existing origin and queue-target controls remained separate green +regressions. + +The minimal correction captures the exact private runtime path prefix from the +first approved scale-set snapshot and carries it through the before/after +scale-set and runner snapshots, session-open, listener hook, ACK, acquisition, +and session-close captures. A same-origin different tenant prefix is rejected +at the marked request boundary before inner transport effects; the exact +origin and private approval marker semantics remain unchanged, and the prefix +is never journaled. Marked session-open request cardinality is reserved before +body forwarding, so a second marked POST is rejected before the inner transport +and cannot create a duplicate live session. + +The focused normal green command was: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineSessionOpenRejectsDuplicateMarkedPOSTBeforeInner|TestBaselineRuntimePathPrefixMismatchStopsBeforeInner|TestPinnedSDKDrainRejectsSnapshotTenantPrefixMismatchBeforeEffects|TestPinnedSDKDrainRejectsSessionTenantPrefixMismatchBeforeListener|TestPinnedSDKDrainRejectsSnapshotOriginMismatchBeforeEffects|TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener|TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody|TestPinnedSDKDrainBindsSessionCloseToSessionOpenOrigin|TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner)$' -count=1 -v -timeout=180s +``` + +It exited 0 in 0.481s. The corresponding focused race command exited 0 in +1.528s with no race report. The new boundary matrix covers same-origin +foreign-prefix scale-set, runner, session-open, acquisition and close requests +and asserts zero inner calls; the duplicate session-open regression asserts +exactly one inner call. + +The complete offline package normal run exited 0 in 32.259s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=240s +``` + +The complete package race run exited 0 in 42.067s with no race report: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=300s +``` + +The tagged paired-terminal preservation matrix exited 0 in 14.716s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture ./livecanary -run '^(TestPairedTerminalActualJournalsFinalize|TestPairedTerminalCompletionCadenceAndReceiptSeparation|TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' -count=1 -v -timeout=240s +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, `bash +scripts/gofmt.sh check`, and `git diff --check` each exited 0. The exact +tested implementation head is +`1f12e11309b273ca54b2b222f702539425ac91b1` (`fix G01 runtime tenant binding +and session cardinality`); the evidence append is documentation-only and +follows that head. No live runner, workflow, credential, App, Keychain, +launchd, Docker, Lima, ScaleSet, JIT, cleanup, or canary operation was run. + +Files changed: `experiments/g01-scaleset/livecanary/baseline_listener.go`, +`baseline_terminal.go`, `baseline_wire.go`, `drain.go`, `drain_driver.go`, +`drain_p1_followup_test.go`, `sdk.go`, `sdk_integration_test.go`, and this +evidence file. + +The remaining live gap is the separately authorized G01 canary/evidence gate; +exact-head Codex review, CI and merge remain coordinator-owned. Rollback is a +focused `git revert --no-edit 1f12e11309b273ca54b2b222f702539425ac91b1` +followed, if needed, by a separate revert of this evidence append; retain the +current journal, reservation and uncertainty, and never reset, erase, replay, +or run live cleanup. From a1f20770dd4f13d3a5979489607447503cf82ff4 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 03:28:12 +0900 Subject: [PATCH 37/63] fix G01 marked ACK and close request boundaries --- .../livecanary/baseline_listener.go | 2 +- .../g01-scaleset/livecanary/baseline_wire.go | 66 ++++- .../g01-scaleset/livecanary/drain_driver.go | 30 ++- .../livecanary/drain_p1_followup_test.go | 228 +++++++++++++++++- .../livecanary/sdk_integration_test.go | 13 +- 5 files changed, 326 insertions(+), 13 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_listener.go b/experiments/g01-scaleset/livecanary/baseline_listener.go index 4aee643..32df113 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener.go @@ -165,7 +165,7 @@ func (b *baselineListener) finish(r baselineRecord, known bool) (controllerRecor return ref, nil } func (b *baselineListener) wire(stage string) *baselineWireCapture { - return &baselineWireCapture{stage: stage, setID: b.setID, organization: b.approval.Organization, owner: b.approval.setName(), queue: b.queue, origin: b.origin, runtimePathPrefix: b.runtimePathPrefix, runtimePathPrefixSet: b.runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} + return &baselineWireCapture{stage: stage, setID: b.setID, organization: b.approval.Organization, owner: b.approval.setName(), sessionID: b.sessionID, queue: b.queue, origin: b.origin, runtimePathPrefix: b.runtimePathPrefix, runtimePathPrefixSet: b.runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(b.approval, b.api.drainEndpointHost())} } func (b *baselineListener) capturedOrigin() string { diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 5b51164..f110dc7 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -133,15 +133,11 @@ func (c *baselineWireCapture) target(r *http.Request) bool { if r == nil || r.URL == nil || r.URL.Fragment != "" || r.URL.User != nil || r.URL.EscapedPath() != r.URL.Path { return false } - if c.stage == "poll" || c.stage == "ack" { + if c.stage == "poll" { u, e := url.Parse(c.queue) if e != nil { return false } - if c.stage == "ack" { - u.Path += "/" + strconv.Itoa(c.cursor) - return r.Method == "DELETE" && r.URL.String() == u.String() - } if c.cursor > 0 { q := u.Query() q.Set("lastMessageId", strconv.Itoa(c.cursor)) @@ -149,6 +145,24 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } return r.Method == "GET" && r.URL.String() == u.String() } + if c.stage == "ack" { + if !c.markedDeleteIdentityReady() || c.queue == "" || c.cursor <= 0 { + return false + } + u, err := url.Parse(c.queue) + if err != nil || u.Fragment != "" || u.User != nil || u.EscapedPath() != u.Path { + return false + } + queueOrigin, ok := baselineRequestOrigin(u) + if !ok || queueOrigin != c.origin { + return false + } + if len(c.allowedHosts) > 0 && !baselineOriginAllowed(u, c.allowedHosts) { + return false + } + u.Path += "/" + strconv.Itoa(c.cursor) + return r.Method == http.MethodDelete && r.URL.String() == u.String() + } suffix := "/runnerscalesets/" + strconv.Itoa(c.setID) + "/" if c.stage == "set-observe" || c.stage == "terminal-set" || c.stage == "terminal-set-recheck" || c.stage == "terminal-set-absence" || c.stage == "terminal-set-delete" { if !c.snapshotOriginAllowed(r) { @@ -162,7 +176,7 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } if c.stage == "terminal-session-close" { origin, ok := baselineRequestOrigin(r.URL) - return c.origin != "" && ok && origin == c.origin && c.sessionID != "" && r.Method == "DELETE" && c.runtimeRequestTarget(r, "sessions/"+c.sessionID) && baselineExactAPIVersionQuery(r.URL.RawQuery) + return c.markedDeleteIdentityReady() && ok && origin == c.origin && (len(c.allowedHosts) == 0 || baselineOriginAllowed(r.URL, c.allowedHosts)) && r.Method == http.MethodDelete && c.runtimeRequestTarget(r, "sessions/"+c.sessionID) && baselineExactAPIVersionQuery(r.URL.RawQuery) } if c.stage == "session-open" { if len(c.allowedHosts) > 0 && !baselineOriginAllowed(r.URL, c.allowedHosts) { @@ -185,6 +199,29 @@ func (c *baselineWireCapture) target(r *http.Request) bool { return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && baselineExactAPIVersionQuery(r.URL.RawQuery) } +// markedDeleteIdentityReady requires the private identity captured by the +// approved G01 session and snapshot sequence before a marked DELETE can be +// forwarded. The fields are never serialized; they bind the physical request +// to the one-shot operation that created the capture. +func (c *baselineWireCapture) markedDeleteIdentityReady() bool { + return c != nil && c.setID > 0 && c.sessionID != "" && c.origin != "" && c.runtimePathPrefixSet && c.runtimePathPrefix != "" +} + +func (c *baselineWireCapture) reserveOneShot(req *http.Request) error { + c.mu.Lock() + if c.requestCount != 0 || c.invalid { + c.invalid = true + c.mu.Unlock() + if req != nil && req.Body != nil { + _ = req.Body.Close() + } + return ErrRemote + } + c.requestCount = 1 + c.mu.Unlock() + return nil +} + func (c *baselineWireCapture) runtimeRequestTarget(r *http.Request, tail string) bool { if c == nil || r == nil || r.URL == nil { return false @@ -354,6 +391,21 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { if c == nil { return nil } + if c.stage == "ack" { + if !c.target(req) { + return c.rejectRequest(req) + } + return c.reserveOneShot(req) + } + if c.stage == "terminal-session-close" { + // A close DELETE is a marked terminal effect, not a snapshot read. It + // must reject every mismatch, including a route that does not contain + // the runnerscalesets family, before the inner transport is called. + if !c.markedDeleteIdentityReady() || !c.target(req) { + return c.rejectRequest(req) + } + return c.reserveOneShot(req) + } if c.stage == "session-open" { if !c.target(req) { if c.sessionOpenBootstrapTarget(req) { @@ -397,7 +449,7 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { c.mu.Unlock() return nil } - if c.stage == "set-observe" || c.stage == "runner-observe" || strings.HasPrefix(c.stage, "terminal-set") || c.stage == "terminal-session-close" { + if c.stage == "set-observe" || c.stage == "runner-observe" || strings.HasPrefix(c.stage, "terminal-set") { if !snapshotRequestCandidate(req) { return nil } diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 0864a0c..e0af5de 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -132,13 +132,35 @@ func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error if c.hook != nil { c.hook.mu.Lock() queue := c.hook.target + origin := c.hook.origin runtimePathPrefix := c.hook.runtimePathPrefix runtimePathPrefixSet := c.hook.runtimePathPrefixSet c.hook.mu.Unlock() if !runtimePathPrefixSet { return Event{}, c.reject("ack") } - wire = &baselineWireCapture{stage: "ack", queue: queue, cursor: id, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet} + setID := c.setID + sessionID := c.sessionID + if c.inner != nil { + current := c.inner.Session() + if setID <= 0 && current.RunnerScaleSet != nil { + setID = current.RunnerScaleSet.ID + } + if sessionID == "" { + sessionID = current.SessionID.String() + } + } + apiHost := "" + if c.d != nil { + if reader, ok := c.d.API.(drainEndpointHostReader); ok { + apiHost = reader.drainEndpointHost() + } + } + var approval Approval + if c.d != nil { + approval = c.d.Approval + } + wire = &baselineWireCapture{stage: "ack", setID: setID, sessionID: sessionID, queue: queue, cursor: id, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(approval, apiHost)} call = wire.context(call) } if err := c.inner.DeleteMessage(call, id); err != nil { @@ -441,7 +463,11 @@ func (d *Driver) drain(ctx context.Context, setID int) error { if origin == "" || !runtimePathPrefixSet { return Event{}, ErrQuarantine } - wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet} + allowedHosts := []string(nil) + if reader, ok := d.API.(drainEndpointHostReader); ok { + allowedHosts = baselineWireAllowedHosts(d.Approval, reader.drainEndpointHost()) + } + wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: allowedHosts} call = wire.context(call) if err := session.Close(call); err != nil { return Event{}, err diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index b813091..248000c 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -466,7 +466,7 @@ func TestBaselineAcquireOriginMismatchStopsBeforeInner(t *testing.T) { } func TestBaselineSessionCloseTargetRequiresExactOrigin(t *testing.T) { - capture := &baselineWireCapture{stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://actions.example:443"} + capture := &baselineWireCapture{stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://actions.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true} for _, tc := range []struct { name string target string @@ -492,6 +492,232 @@ func TestBaselineSessionCloseTargetRequiresExactOrigin(t *testing.T) { } } +func TestBaselineMarkedACKMismatchStopsBeforeInner(t *testing.T) { + const ( + queue = "https://api.example/tenant/v2/queue?proof=fixture" + origin = "https://api.example:443" + prefix = "/tenant/v2" + ) + for _, tc := range []struct { + name string + target string + }{ + {name: "wrong message", target: "https://api.example/tenant/v2/queue/42?proof=fixture"}, + {name: "wrong path", target: "https://api.example/tenant/v2/other/41?proof=fixture"}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "ack", setID: 7, sessionID: "session", queue: queue, cursor: 41, + origin: origin, runtimePathPrefix: prefix, runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodDelete, tc.target, nil) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched marked ACK error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("mismatched marked ACK reached inner transport: calls=%d", innerCalls) + } + if capture.requestObserved() { + t.Fatal("mismatched marked ACK was marked observed") + } + }) + } +} + +func TestBaselineMarkedACKRequiresIdentityAndOneShotCardinality(t *testing.T) { + const target = "https://api.example/tenant/v2/queue/41?proof=fixture" + for _, tc := range []struct { + name string + mutate func(*baselineWireCapture) + }{ + {name: "missing origin", mutate: func(c *baselineWireCapture) { c.origin = "" }}, + {name: "missing tenant prefix", mutate: func(c *baselineWireCapture) { c.runtimePathPrefix = ""; c.runtimePathPrefixSet = false }}, + {name: "missing scale set", mutate: func(c *baselineWireCapture) { c.setID = 0 }}, + {name: "missing session", mutate: func(c *baselineWireCapture) { c.sessionID = "" }}, + {name: "unapproved queue origin", mutate: func(c *baselineWireCapture) { c.queue = "https://other.example/tenant/v2/queue?proof=fixture" }}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "ack", setID: 7, sessionID: "session", queue: "https://api.example/tenant/v2/queue?proof=fixture", cursor: 41, + origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + } + tc.mutate(capture) + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodDelete, target, nil) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("invalid marked ACK identity error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("invalid marked ACK identity reached inner transport: calls=%d", innerCalls) + } + }) + } + + capture := &baselineWireCapture{ + stage: "ack", setID: 7, sessionID: "session", queue: "https://api.example/tenant/v2/queue?proof=fixture", cursor: 41, + origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + for i := 0; i < 2; i++ { + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodDelete, target, nil) + if err != nil { + t.Fatal(err) + } + _, err = transport.RoundTrip(req) + if i == 0 && err != nil { + t.Fatalf("valid marked ACK = %v, want forwarding", err) + } + if i == 1 && !errors.Is(err, ErrRemote) { + t.Fatalf("duplicate marked ACK = %v, want remote rejection", err) + } + } + if innerCalls != 1 { + t.Fatalf("marked ACK physical cardinality = %d, want one inner call", innerCalls) + } +} + +func TestBaselineMarkedSessionCloseMismatchStopsBeforeInner(t *testing.T) { + const ( + origin = "https://api.example:443" + prefix = "/tenant/v2" + ) + for _, tc := range []struct { + name string + path string + }{ + {name: "route family omitted", path: "/tenant/v2/_apis/runtime/sessions/session?api-version=6.0-preview"}, + {name: "wrong session path", path: "/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/other?api-version=6.0-preview"}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "terminal-session-close", setID: 7, sessionID: "session", origin: origin, + runtimePathPrefix: prefix, runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodDelete, "https://api.example"+tc.path, nil) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched marked session-close error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("mismatched marked session-close reached inner transport: calls=%d", innerCalls) + } + if capture.requestObserved() { + t.Fatal("mismatched marked session-close was marked observed") + } + }) + } +} + +func TestBaselineMarkedSessionCloseRequiresIdentityAndOneShotCardinality(t *testing.T) { + const target = "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview" + for _, tc := range []struct { + name string + mutate func(*baselineWireCapture) + }{ + {name: "missing origin", mutate: func(c *baselineWireCapture) { c.origin = "" }}, + {name: "missing tenant prefix", mutate: func(c *baselineWireCapture) { c.runtimePathPrefix = ""; c.runtimePathPrefixSet = false }}, + {name: "missing scale set", mutate: func(c *baselineWireCapture) { c.setID = 0 }}, + {name: "missing session", mutate: func(c *baselineWireCapture) { c.sessionID = "" }}, + {name: "wrong origin", mutate: func(c *baselineWireCapture) { c.origin = "https://other.example:443" }}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + } + tc.mutate(capture) + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodDelete, target, nil) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("invalid marked session-close identity error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("invalid marked session-close identity reached inner transport: calls=%d", innerCalls) + } + }) + } + + capture := &baselineWireCapture{ + stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + for i := 0; i < 2; i++ { + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodDelete, target, nil) + if err != nil { + t.Fatal(err) + } + _, err = transport.RoundTrip(req) + if i == 0 && err != nil { + t.Fatalf("valid marked session-close = %v, want forwarding", err) + } + if i == 1 && !errors.Is(err, ErrRemote) { + t.Fatalf("duplicate marked session-close = %v, want remote rejection", err) + } + } + if innerCalls != 1 { + t.Fatalf("marked session-close physical cardinality = %d, want one inner call", innerCalls) + } +} + +func TestUnmarkedDeletePreservesInnerTransport(t *testing.T) { + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequest(http.MethodDelete, "http://unmarked.invalid/arbitrary-target", nil) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("unmarked delete = %v, want forwarding", err) + } + if innerCalls != 1 { + t.Fatalf("unmarked delete inner calls = %d, want one", innerCalls) + } +} + func TestBaselineAcquireForwardingBodySurvivesAsyncRoundTripClose(t *testing.T) { capture := &baselineWireCapture{stage: "acquire", setID: 7, origin: "https://api.example:443", requestIDs: []int64{41}, allowedHosts: []string{"api.example"}} release := make(chan struct{}) diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 0ca50ec..fa4c3d3 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -677,8 +677,8 @@ func TestPinnedSDKDrainBindsACKToPhysicalDelete(t *testing.T) { if err := client.DeleteMessage(context.Background(), message.MessageID); !errors.Is(err, ErrQuarantine) { t.Fatalf("wrong physical ACK = %v, want quarantine", err) } - if fixture.acks.Load() != 1 { - t.Fatalf("wrong physical ACK did not reach successful fixture endpoint: ack=%d", fixture.acks.Load()) + if fixture.acks.Load() != 0 { + t.Fatalf("wrong physical ACK reached fixture endpoint: ack=%d", fixture.acks.Load()) } } @@ -829,6 +829,15 @@ func TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete(t *testing.T) { } }, }, + { + name: "route family omitted", + mutate: func(req *http.Request) { + if req.Method == http.MethodDelete && strings.Contains(req.URL.Path, "/sessions/") { + req.URL.Path = strings.Replace(req.URL.Path, "/runnerscalesets/7", "", 1) + req.URL.RawPath = "" + } + }, + }, } { t.Run(tc.name, func(t *testing.T) { fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{mutateRequest: tc.mutate}) From a0702639247bc1f97f030d3c9adbecdc38af37da Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 03:29:26 +0900 Subject: [PATCH 38/63] test G01 DELETE boundaries report inner calls --- .../livecanary/drain_p1_followup_test.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 248000c..b72fc98 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -520,12 +520,13 @@ func TestBaselineMarkedACKMismatchStopsBeforeInner(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { - t.Fatalf("mismatched marked ACK error = %v, want remote rejection", err) - } + _, err = transport.RoundTrip(req) if innerCalls != 0 { t.Fatalf("mismatched marked ACK reached inner transport: calls=%d", innerCalls) } + if !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched marked ACK error = %v, want remote rejection", err) + } if capture.requestObserved() { t.Fatal("mismatched marked ACK was marked observed") } @@ -624,12 +625,13 @@ func TestBaselineMarkedSessionCloseMismatchStopsBeforeInner(t *testing.T) { if err != nil { t.Fatal(err) } - if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { - t.Fatalf("mismatched marked session-close error = %v, want remote rejection", err) - } + _, err = transport.RoundTrip(req) if innerCalls != 0 { t.Fatalf("mismatched marked session-close reached inner transport: calls=%d", innerCalls) } + if !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched marked session-close error = %v, want remote rejection", err) + } if capture.requestObserved() { t.Fatal("mismatched marked session-close was marked observed") } From 1e19ec56acbeda044acaf80c2b8ada0e5d66c192 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 03:40:47 +0900 Subject: [PATCH 39/63] docs G01 marked DELETE boundary evidence --- docs/evidence/g01-idle-drain.md | 90 +++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 6f78ee1..af6fd62 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1418,6 +1418,96 @@ Rollback is a focused normal `git revert --no-edit` of this correction commit; retain the journal, reservation and uncertainty, and never reset, erase, replay, or run live cleanup. +### Exact-head follow-up: marked ACK and session-close DELETE pre-forward fences + +Date: 2026-09-13. This correction started from exact head +`1bb0c878fe97b81cae61f3416cd074670ec2292d` for PR #72. The fresh Codex +findings were [marked ACK DELETE validation](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997097883) +and [marked session-close DELETE fall-through](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997097890). +No Project, Issue, goal, dependency, branch-ownership or live-operation state +was changed. + +#### Red-first reproductions + +The meaningful red-first command ran after adding the behavioral regressions +and before changing production code: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineMarkedACKMismatchStopsBeforeInner|TestBaselineMarkedACKRequiresIdentityAndOneShotCardinality|TestBaselineMarkedSessionCloseMismatchStopsBeforeInner|TestBaselineMarkedSessionCloseRequiresIdentityAndOneShotCardinality|TestUnmarkedDeletePreservesInnerTransport)$' -count=1 -v -timeout=120s +``` + +It exited 1 as intended. Wrong ACK message/path and the omitted +`runnerscalesets` session-close route returned nil instead of a pre-forward +`ErrRemote`; duplicate marked ACK and close requests were also accepted by the +old transport boundary, while the unmarked DELETE control passed. The +regressions count inner calls and retain only fixed outcomes/counters; no +request body, bearer, queue URL, response error or private log is recorded. + +#### Corrections and boundary evidence + +Marked ACK captures now require the exact captured queue URL plus message ID, +canonical expected origin, an approved queue host when an allowlist is +available, a captured tenant prefix, and positive scale-set/session identity +before reserving the one-shot physical request. The request reservation is +made before the inner SDK transport and rejects a second physical DELETE; +`baseline_listener` now carries its session identity into the same marked +capture, and the drain client carries origin, tenant prefix, set/session ID and +approved host metadata into ACK captures. Existing ACK-before-acquisition and +response-status checks remain unchanged. + +Marked session-close DELETEs now take a dedicated branch before +`snapshotRequestCandidate`, so a mismatch cannot fall through merely because +its route omits the `runnerscalesets` family. The branch requires exact HTTPS +origin, approved host, tenant prefix, scale-set ID, session ID, method, route, +API-version query and one-shot cardinality before forwarding; unmarked +non-G01 requests retain the prior inner-transport behavior. + +The focused green normal command passed in 0.478s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineMarkedACKMismatchStopsBeforeInner|TestBaselineMarkedACKRequiresIdentityAndOneShotCardinality|TestBaselineMarkedSessionCloseMismatchStopsBeforeInner|TestBaselineMarkedSessionCloseRequiresIdentityAndOneShotCardinality|TestUnmarkedDeletePreservesInnerTransport|TestPinnedSDKDrainBindsACKToPhysicalDelete|TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete)$' -count=1 -v -timeout=240s +``` + +The corresponding focused race suite passed in 1.571s with no race +diagnostics. The pinned-SDK controls now assert zero fixture calls for a +rewritten ACK and for wrong session-close session/set/route-family paths; the +direct boundary matrix covers missing/wrong origin, tenant prefix, set/session +identity and duplicate physical requests, while the unmarked control asserts +one forwarded inner call. + +The full offline package and related preservation checks passed after the +correction: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=360s +GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture ./livecanary -run '^(TestPairedTerminalActualJournalsFinalize|TestPairedTerminalCompletionCadenceAndReceiptSeparation|TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' -count=1 -v -timeout=300s +cd ../.. +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +bash scripts/gofmt.sh check +git diff --check +``` + +The normal package exited 0 in 29.692s, the race package exited 0 in 45.666s +with no race diagnostics, vet and the tagged paired-terminal matrix exited 0, +and the two-module offline gate exited 0; format/diff checks were clean. The +exact implementation/test head is +`a0702639247bc1f97f030d3c9adbecdc38af37da`, comprising source correction +`a1f20770dd4f13d3a5979489607447503cf82ff4` and test assertion refinement +`a0702639247bc1f97f030d3c9adbecdc38af37da`. + +Rollback is recoverable with focused normal +`git revert --no-edit a0702639247bc1f97f030d3c9adbecdc38af37da a1f20770dd4f13d3a5979489607447503cf82ff4`; +the documentation-only append can be reverted separately. No live GitHub App, +runner, Scale Set, JIT, workflow, Docker/Lima, Keychain, launchd, credential, +network, cleanup or rollback operation was performed. The remaining gap is +the coordinator-owned exact-head Codex review and required CI/live G01 gate; +this worker did not request review or merge. + ## Remaining gate and rollback The live G01 gate remains unresolved until a separately authorized run uses an From 44524b9f290689378348951a722d7eb2ffb3d070 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 04:13:31 +0900 Subject: [PATCH 40/63] fix G01 marked request Host boundary --- .../g01-scaleset/livecanary/baseline_wire.go | 10 ++ experiments/g01-scaleset/livecanary/drain.go | 3 + .../livecanary/drain_p1_followup_test.go | 97 +++++++++++++++++++ 3 files changed, 110 insertions(+) diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index f110dc7..7b2c2df 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -347,9 +347,19 @@ func baselineWireAllowedHosts(a Approval, apiHost string) []string { // process. Only explicitly marked G01 runtime requests are inspected. type baselineRequestCaptureTransport struct{ inner http.RoundTripper } +// baselineRequestHostMatchesURL fences Request.Host, which overrides the +// physical HTTP Host header. The SDK normally leaves it empty; an explicitly +// supplied value is accepted only when it equals the URL's canonical host. +func baselineRequestHostMatchesURL(req *http.Request) bool { + return req != nil && req.URL != nil && (req.Host == "" || req.Host == req.URL.Host) +} + func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) { c, _ := req.Context().Value(baselineWireKey{}).(*baselineWireCapture) if c != nil { + if !baselineRequestHostMatchesURL(req) { + return nil, c.rejectRequest(req) + } if err := c.captureRequest(req); err != nil { return nil, err } diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 5e3f906..3b406ce 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -553,6 +553,9 @@ func (h *drainPollHook) RoundTrip(req *http.Request) (*http.Response, error) { if approved != h { return h.rejectPoll() } + if !baselineRequestHostMatchesURL(req) { + return h.rejectPoll() + } h.mu.Lock() h.pollAttempts++ attempt := h.pollAttempts diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index b72fc98..b105d51 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -344,6 +344,7 @@ func TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner(t *testing.T) { mutate func(*http.Request) }{ {name: "wrong host", mutate: func(req *http.Request) { req.URL.Host = "other.invalid" }}, + {name: "overridden Host", mutate: func(req *http.Request) { req.Host = "other.invalid" }}, {name: "wrong path", mutate: func(req *http.Request) { req.URL.Path = "/wrong-queue"; req.URL.RawPath = "" }}, {name: "wrong origin scheme", mutate: func(req *http.Request) { req.URL.Scheme = "http" }}, {name: "wrong queue proof", mutate: func(req *http.Request) { @@ -356,6 +357,8 @@ func TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner(t *testing.T) { hook := newDrainPollHook(target) hook.mu.Lock() hook.origin = "https://fixture.invalid:443" + hook.runtimePathPrefix = "/tenant/v2" + hook.runtimePathPrefixSet = true hook.mu.Unlock() innerCalls := 0 hook.inner = drainRoundTripper(func(req *http.Request) (*http.Response, error) { @@ -381,6 +384,98 @@ func TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner(t *testing.T) { } } +func TestMarkedRequestHostOverrideStopsBeforeInner(t *testing.T) { + const ( + origin = "https://api.example:443" + prefix = "/tenant/v2" + ) + for _, tc := range []struct { + name string + stage string + method string + target string + body string + }{ + { + name: "session-open", + stage: "session-open", + method: http.MethodPost, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", + body: `{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"g01-test"}`, + }, + { + name: "ACK", + stage: "ack", + method: http.MethodDelete, + target: "https://api.example/tenant/v2/queue/41?proof=fixture", + }, + { + name: "acquisition", + stage: "acquire", + method: http.MethodPost, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", + body: "[41]", + }, + { + name: "session-close", + stage: "terminal-session-close", + method: http.MethodDelete, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", + }, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: tc.stage, setID: 7, owner: "g01-test", organization: "fixture-org", sessionID: "session", + queue: "https://api.example/tenant/v2/queue?proof=fixture", cursor: 41, + origin: origin, runtimePathPrefix: prefix, runtimePathPrefixSet: true, requestIDs: []int64{41}, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, tc.target, strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + req.Host = "other.example" + _, err = transport.RoundTrip(req) + if !errors.Is(err, ErrRemote) { + t.Fatalf("marked %s Host override error = %v, want remote rejection", tc.name, err) + } + if innerCalls != 0 { + t.Fatalf("marked %s Host override reached inner transport: calls=%d", tc.name, innerCalls) + } + if capture.requestObserved() { + t.Fatalf("marked %s Host override was marked observed", tc.name) + } + }) + } +} + +func TestMarkedRequestHostMatchingURLHostPreservesForwarding(t *testing.T) { + capture := &baselineWireCapture{ + stage: "acquire", setID: 7, origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + requestIDs: []int64{41}, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", strings.NewReader("[41]")) + if err != nil { + t.Fatal(err) + } + req.Host = req.URL.Host + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("marked request with URL Host override = %v, want forwarding", err) + } + if innerCalls != 1 { + t.Fatalf("marked request with URL Host override inner calls = %d, want one", innerCalls) + } +} + func TestDrainPollHookRejectsApprovalFromDifferentHook(t *testing.T) { const target = "https://fixture.invalid/queue?proof=fixture" hook := newDrainPollHook(target) @@ -414,6 +509,7 @@ func TestDrainPollHookForwardsUnmarkedNonPollRequest(t *testing.T) { if err != nil { t.Fatal(err) } + req.Host = "override.invalid" if _, err := hook.RoundTrip(req); err != nil { t.Fatalf("unmarked session request: %v", err) } @@ -712,6 +808,7 @@ func TestUnmarkedDeletePreservesInnerTransport(t *testing.T) { if err != nil { t.Fatal(err) } + req.Host = "override.invalid" if _, err := transport.RoundTrip(req); err != nil { t.Fatalf("unmarked delete = %v, want forwarding", err) } From d58224fe9a6b20b45b61eeb839fd84642ec587a2 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 04:18:52 +0900 Subject: [PATCH 41/63] docs G01 Host boundary evidence --- docs/evidence/g01-idle-drain.md | 84 +++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index af6fd62..d57b2e2 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1702,3 +1702,87 @@ focused `git revert --no-edit 1f12e11309b273ca54b2b222f702539425ac91b1` followed, if needed, by a separate revert of this evidence append; retain the current journal, reservation and uncertainty, and never reset, erase, replay, or run live cleanup. + +### Exact-head follow-up: physical Host authority at marked wire boundaries + +Date: 2026-09-13. This correction started from exact head +`1e19ec56acbeda044acaf80c2b8ada0e5d66c192` and addresses the fresh Codex P1 +[request Host override finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997211059). +No review request, merge, Project/Issue/goal/status change, live App, runner, +workflow, GitHub API, credential, or cleanup operation was performed. + +#### Red-first reproductions + +After adding the marked-operation and marked-poll Host regressions, but before +the source correction, this focused command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestMarkedRequestHostOverrideStopsBeforeInner|TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner|TestMarkedRequestHostMatchingURLHostPreservesForwarding|TestUnmarkedDeletePreservesInnerTransport)$' -count=1 -v -timeout=120s +``` + +The overridden Host poll case reached its inner transport once, and all four +marked session-open, ACK, acquisition and session-close cases returned nil and +would have reached their inner transport; the exact-Host and unmarked controls +passed. The test retained only fixed error categories and inner-call counters. + +#### Minimal correction and green evidence + +The final baseline wire boundary now accepts only an empty `req.Host` (the +normal SDK form) or exact equality with `req.URL.Host`; a mismatch is rejected +before any marked session-open, ACK, acquisition or session-close inner call. +The marked poll hook applies the same check before its inner transport, while +unmarked requests retain their prior forwarding behavior. The explicit +`req.Host == req.URL.Host` marked control and unmarked overridden-Host controls +pass. + +The focused normal command exited 0 in 0.517s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestDrain|TestPinnedSDKDrain|TestBaseline.*(Acquire|SessionOpen|Snapshot|SessionClose)|TestMarkedRequestHost.*|TestUnmarkedDeletePreservesInnerTransport)$' -count=1 -timeout=300s +``` + +The corresponding focused race command exited 0 in 1.912s with no race +diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestDrain|TestPinnedSDKDrain|TestBaseline.*(Acquire|SessionOpen|Snapshot|SessionClose)|TestMarkedRequestHost.*|TestUnmarkedDeletePreservesInnerTransport)$' -count=1 -timeout=300s +``` + +The complete `livecanary` package normal run exited 0 in 30.030s, and the +complete race run exited 0 in 41.306s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=300s +``` + +The repository offline gate then exited 0 and printed +`offline experiment checks passed: 2 module(s)`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`GOWORK=off GOTOOLCHAIN=go1.26.8 bash ../../scripts/gofmt.sh check`, and +`git diff --check` each exited 0. The final diff secret/private-path scan +exited 0 and printed `diff secret/private-path scan passed`: + +```text +set -e +path_pattern="$(printf '/%s/|/%s/' Users home)" +if git diff --text | rg -n "(${path_pattern}|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})"; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The source/test correction is commit +`44524b9f290689378348951a722d7eb2ffb3d070`; this evidence append is the +documentation-only commit immediately after it. Rollback is recoverable with +`git revert --no-edit 44524b9f290689378348951a722d7eb2ffb3d070`; the evidence +append can be reverted separately if required. No live rollback was run, and +the unresolved live G01 gate, exact-head Codex review, CI and merge remain +coordinator-owned. From bf45b498db8bd35150d0f75181336663ea4cb027 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 04:52:52 +0900 Subject: [PATCH 42/63] fix G01 marked opaque and snapshot boundaries --- .../livecanary/baseline_terminal.go | 4 +- .../g01-scaleset/livecanary/baseline_wire.go | 7 +- .../g01-scaleset/livecanary/drain_driver.go | 4 +- .../livecanary/drain_p1_followup_test.go | 175 ++++++++++++++++++ 4 files changed, 184 insertions(+), 6 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal.go b/experiments/g01-scaleset/livecanary/baseline_terminal.go index 018d349..ee0f846 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal.go @@ -135,7 +135,7 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { if !runtimePathPrefixSet { return ErrQuarantine } - capture := &baselineWireCapture{stage: stage, setID: s.setID, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} + capture := &baselineWireCapture{stage: stage, setID: s.setID, organization: s.approval.Organization, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) set, e := s.captured.GetScaleSet(capture.context(ctx), s.setID) callErr = e @@ -180,7 +180,7 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { if !runtimePathPrefixSet { return ErrQuarantine } - capture := &baselineWireCapture{stage: stage, setID: s.setID, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} + capture := &baselineWireCapture{stage: stage, setID: s.setID, organization: s.approval.Organization, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) callErr = s.captured.DeleteScaleSet(capture.context(ctx), s.setID) cancel() diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 7b2c2df..e90edfe 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -351,7 +351,7 @@ type baselineRequestCaptureTransport struct{ inner http.RoundTripper } // physical HTTP Host header. The SDK normally leaves it empty; an explicitly // supplied value is accepted only when it equals the URL's canonical host. func baselineRequestHostMatchesURL(req *http.Request) bool { - return req != nil && req.URL != nil && (req.Host == "" || req.Host == req.URL.Host) + return req != nil && req.URL != nil && req.URL.Opaque == "" && (req.Host == "" || req.Host == req.URL.Host) } func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) { @@ -461,7 +461,10 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { } if c.stage == "set-observe" || c.stage == "runner-observe" || strings.HasPrefix(c.stage, "terminal-set") { if !snapshotRequestCandidate(req) { - return nil + if c.sessionOpenBootstrapTarget(req) { + return nil + } + return c.rejectRequest(req) } if !c.target(req) { return c.rejectRequest(req) diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index e0af5de..6ff076d 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -260,7 +260,7 @@ func (d *Driver) drainSnapshotWithBinding(ctx context.Context, setID int, stage, if err := d.effect(ctx, "observe-owned", nil, func(call context.Context) (Event, error) { var err error if setWire { - wire := &baselineWireCapture{stage: "set-observe", setID: setID, origin: expectedOrigin, runtimePathPrefix: expectedRuntimePathPrefix, runtimePathPrefixSet: expectedRuntimePathPrefix != "", allowedHosts: allowedHosts} + wire := &baselineWireCapture{stage: "set-observe", setID: setID, organization: d.Approval.Organization, origin: expectedOrigin, runtimePathPrefix: expectedRuntimePathPrefix, runtimePathPrefixSet: expectedRuntimePathPrefix != "", allowedHosts: allowedHosts} set, err = setReader.drainGetScaleSet(call, setID, wire) wireSet, status := wire.setFacts() wireOrigin := wire.requestOrigin() @@ -299,7 +299,7 @@ func (d *Driver) drainSnapshotWithBinding(ctx context.Context, setID int, stage, if err := d.effect(ctx, "observe-runner", nil, func(call context.Context) (Event, error) { var err error if runnerWire { - wire := &baselineWireCapture{stage: "runner-observe", runnerName: d.Approval.workerName(), origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefix != "", allowedHosts: allowedHosts} + wire := &baselineWireCapture{stage: "runner-observe", runnerName: d.Approval.workerName(), organization: d.Approval.Organization, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefix != "", allowedHosts: allowedHosts} runner, err = runnerReader.drainFindRunner(call, d.Approval.workerName(), wire) wireRunner, status := wire.runnerFacts() wireOrigin := wire.requestOrigin() diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index b105d51..6476a1c 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -337,6 +337,181 @@ func TestBaselineRuntimePathPrefixMismatchStopsBeforeInner(t *testing.T) { } } +func TestMarkedRequestOpaqueStopsBeforeInner(t *testing.T) { + const ( + origin = "https://api.example:443" + prefix = "/tenant/v2" + ) + for _, tc := range []struct { + name string + stage string + method string + target string + body string + }{ + { + name: "set snapshot", + stage: "set-observe", + method: http.MethodGet, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", + }, + { + name: "runner snapshot", + stage: "runner-observe", + method: http.MethodGet, + target: "https://api.example/tenant/v2/_apis/distributedtask/pools/0/agents?agentName=g01-test-worker-1&api-version=6.0-preview", + }, + { + name: "session open", + stage: "session-open", + method: http.MethodPost, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", + body: `{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"g01-test"}`, + }, + { + name: "ACK", + stage: "ack", + method: http.MethodDelete, + target: "https://api.example/tenant/v2/queue/41?proof=fixture", + }, + { + name: "acquisition", + stage: "acquire", + method: http.MethodPost, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", + body: `[41]`, + }, + { + name: "JIT", + stage: "jit", + method: http.MethodPost, + target: "https://api.example/_apis/runtime/runnerscalesets/7/generatejitconfig?api-version=6.0-preview", + body: `{}`, + }, + { + name: "session close", + stage: "terminal-session-close", + method: http.MethodDelete, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", + }, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: tc.stage, setID: 7, organization: "fixture-org", owner: "g01-test", runnerName: "g01-test-worker-1", sessionID: "session", + queue: "https://api.example/tenant/v2/queue?proof=fixture", cursor: 41, + origin: origin, runtimePathPrefix: prefix, runtimePathPrefixSet: true, requestIDs: []int64{41}, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, tc.target, strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + req.URL.Opaque = "opaque-target" + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("marked %s opaque URL error = %v, want remote rejection", tc.name, err) + } + if innerCalls != 0 { + t.Fatalf("marked %s opaque URL reached inner transport: calls=%d", tc.name, innerCalls) + } + if capture.requestObserved() { + t.Fatalf("marked %s opaque URL was marked observed", tc.name) + } + }) + } +} + +func TestMarkedPollOpaqueStopsBeforeInner(t *testing.T) { + const target = "https://fixture.invalid/queue?proof=fixture" + hook := newDrainPollHookWithOriginAndPrefix(target, "https://fixture.invalid:443", "/tenant/v2") + innerCalls := 0 + hook.inner = drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusAccepted, Body: http.NoBody, Request: req}, nil + }) + req, err := http.NewRequestWithContext(hook.markPoll(context.Background()), http.MethodGet, target, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, "1") + req.URL.Opaque = "opaque-target" + if _, err := hook.RoundTrip(req); !errors.Is(err, ErrQuarantine) { + t.Fatalf("marked poll opaque URL error = %v, want quarantine", err) + } + if innerCalls != 0 { + t.Fatalf("marked poll opaque URL reached inner transport: calls=%d", innerCalls) + } +} + +func TestMarkedSnapshotRewriteAllowsOnlyRequiredBootstrap(t *testing.T) { + for _, stage := range []string{"set-observe", "runner-observe", "terminal-set"} { + for _, tc := range []struct { + name string + method string + target string + allow bool + }{ + {name: "registration bootstrap", method: http.MethodPost, target: "https://api.example/orgs/fixture-org/actions/runners/registration-token", allow: true}, + {name: "actions bootstrap", method: http.MethodPost, target: "https://api.example/actions/runner-registration", allow: true}, + {name: "repository bootstrap rewrite", method: http.MethodPost, target: "https://api.example/repos/fixture/repo/actions/runners/registration-token"}, + {name: "dispatch state change", method: http.MethodPost, target: "https://api.example/repos/fixture/repo/dispatches"}, + {name: "session state change", method: http.MethodPost, target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview"}, + {name: "unrelated read", method: http.MethodGet, target: "https://api.example/repos/fixture/repo"}, + } { + t.Run(stage+"/"+tc.name, func(t *testing.T) { + capture := &baselineWireCapture{stage: stage, setID: 7, organization: "fixture-org", origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}} + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, tc.target, nil) + if err != nil { + t.Fatal(err) + } + _, err = transport.RoundTrip(req) + if tc.allow { + if err != nil || innerCalls != 1 { + t.Fatalf("required snapshot bootstrap err=%v inner calls=%d, want one forwarded request", err, innerCalls) + } + if capture.requestObserved() { + t.Fatal("required bootstrap was counted as the snapshot") + } + return + } + if !errors.Is(err, ErrRemote) { + t.Fatalf("non-required snapshot rewrite error = %v, want remote rejection", err) + } + if innerCalls != 0 { + t.Fatalf("non-required snapshot rewrite reached inner transport: calls=%d", innerCalls) + } + }) + } + } +} + +func TestUnmarkedOpaqueRequestPreservesInnerForwarding(t *testing.T) { + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequest(http.MethodPost, "https://api.example/unmarked", nil) + if err != nil { + t.Fatal(err) + } + req.URL.Opaque = "opaque-target" + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("unmarked opaque request = %v, want forwarding", err) + } + if innerCalls != 1 { + t.Fatalf("unmarked opaque request inner calls = %d, want one", innerCalls) + } +} + func TestDrainListenerRejectsMarkedPollTargetMismatchBeforeInner(t *testing.T) { const target = "https://fixture.invalid/queue?proof=fixture" for _, tc := range []struct { From 91aa852cc2f66102aad54c402b7da8e552163828 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 04:54:08 +0900 Subject: [PATCH 43/63] docs G01 opaque and snapshot boundary evidence --- docs/evidence/g01-idle-drain.md | 85 +++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index d57b2e2..8b45ac9 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1786,3 +1786,88 @@ documentation-only commit immediately after it. Rollback is recoverable with append can be reverted separately if required. No live rollback was run, and the unresolved live G01 gate, exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: opaque marked URLs and snapshot bootstrap allowlist + +Date: 2026-09-13. This correction started from exact head +`d58224fe9a6b20b45b61eeb839fd84642ec587a2` and addresses the fresh Codex P1 +findings [non-empty marked URL.Opaque forwarding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997312693) +and [marked snapshot non-target rewrite](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997312695). +No review request, merge, Project/Issue/goal/status change, live App, runner, +workflow, GitHub API, credential, cleanup or canary operation was performed. + +#### Red-first reproductions + +After adding the two behavioral regressions and before changing source, this +focused command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestMarkedRequestOpaqueStopsBeforeInner|TestMarkedPollOpaqueStopsBeforeInner|TestMarkedSnapshotRewriteAllowsOnlyRequiredBootstrap|TestUnmarkedOpaqueRequestPreservesInnerForwarding)$' -count=1 -v -timeout=120s +``` + +The marked set/runner snapshot, session-open, ACK, acquisition, JIT, +session-close and poll cases with non-empty `URL.Opaque` returned nil and +reached the inner transport; the repository-token and dispatch snapshot +rewrites also returned nil and reached the inner transport. The exact required +bootstrap routes and unmarked opaque control passed. The regressions retain +only fixed error categories and inner-call counters; no request body, token, +URL, response error or private log is recorded. + +#### Minimal correction and green evidence + +The final marked request boundary now rejects every non-empty `URL.Opaque` +before forwarding, covering both the baseline marked transport and the marked +drain poll hook through the shared hierarchical-URL/Host check. Snapshot +captures now allow non-candidate requests only when they match the existing +exact approved registration-token or Actions runner-registration bootstrap +allowlist; all other marked non-target requests, including state-changing or +wrong-scope routes, reject before the inner transport. The production snapshot +captures carry the approved organization needed to validate those bootstrap +routes. Unmarked requests remain outside the marked check and retain prior +forwarding behavior. + +The focused normal command exited 0 in 0.513s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestMarkedRequestOpaqueStopsBeforeInner|TestMarkedPollOpaqueStopsBeforeInner|TestMarkedSnapshotRewriteAllowsOnlyRequiredBootstrap|TestUnmarkedOpaqueRequestPreservesInnerForwarding)$' -count=1 -v -timeout=120s +``` + +The corresponding focused race command exited 0 in 1.542s with no race +diagnostics. The complete `livecanary` package normal run exited 0 in 28.951s, +and the complete race run exited 0 in 41.493s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestMarkedRequestOpaqueStopsBeforeInner|TestMarkedPollOpaqueStopsBeforeInner|TestMarkedSnapshotRewriteAllowsOnlyRequiredBootstrap|TestUnmarkedOpaqueRequestPreservesInnerForwarding)$' -count=1 -v -timeout=120s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=360s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=420s +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check`, and `git diff --check` +each exited 0. The repository offline gate exited 0 and printed +`offline experiment checks passed: 2 module(s)`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +The final diff secret/private-path scan exited 0 and printed +`diff secret/private-path scan passed`: + +```text +set -e +path_pattern="$(printf '/%s/|/%s/' Users home)" +if git diff --text | rg -n "(${path_pattern}|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})"; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The exact tested implementation head is +`bf45b498db8bd35150d0f75181336663ea4cb027` (`fix G01 marked opaque and snapshot boundaries`), +and this evidence append is documentation-only. Rollback is recoverable with +`git revert --no-edit bf45b498db8bd35150d0f75181336663ea4cb027`; this evidence +append can be reverted separately if required. No live rollback was run, and +the unresolved live G01 gate, exact-head Codex review, CI and merge remain +coordinator-owned. From 3be170f82e4b173baf808f22e3637dfd533fe7d4 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 05:24:43 +0900 Subject: [PATCH 44/63] fix(g01): bind marked session origin before forwarding --- .../livecanary/baseline_listener.go | 4 ++ .../g01-scaleset/livecanary/baseline_wire.go | 3 +- .../livecanary/drain_p1_followup_test.go | 51 +++++++++++++++++++ experiments/g01-scaleset/livecanary/sdk.go | 3 +- .../livecanary/sdk_integration_test.go | 3 ++ 5 files changed, 62 insertions(+), 2 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_listener.go b/experiments/g01-scaleset/livecanary/baseline_listener.go index 32df113..2e43195 100644 --- a/experiments/g01-scaleset/livecanary/baseline_listener.go +++ b/experiments/g01-scaleset/livecanary/baseline_listener.go @@ -245,12 +245,16 @@ func (b *baselineListener) initialize() error { cancel() r.Set = c.set r.HTTPStatus = c.status + beforeOrigin := c.requestOrigin() prefix, prefixKnown := c.requestRuntimePathPrefix() known := c.observed() && prefixKnown && r.Set.eligible(b.approval, b.setID) && ((callErr != nil && b.ctx.Err() != nil) || (callErr == nil && set != nil && set.ID == r.Set.ID && set.Name == r.Set.Name && set.RunnerGroupID == r.Set.GroupID && set.RunnerSetting.DisableUpdate && r.Set.Statistics.matches(set.Statistics))) if prefixKnown { b.runtimePathPrefix = prefix b.runtimePathPrefixSet = true } + if c.observed() && beforeOrigin != "" { + b.origin = beforeOrigin + } if _, err = b.finish(r, known); err != nil { return err } diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index e90edfe..bcbf7c9 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -179,7 +179,8 @@ func (c *baselineWireCapture) target(r *http.Request) bool { return c.markedDeleteIdentityReady() && ok && origin == c.origin && (len(c.allowedHosts) == 0 || baselineOriginAllowed(r.URL, c.allowedHosts)) && r.Method == http.MethodDelete && c.runtimeRequestTarget(r, "sessions/"+c.sessionID) && baselineExactAPIVersionQuery(r.URL.RawQuery) } if c.stage == "session-open" { - if len(c.allowedHosts) > 0 && !baselineOriginAllowed(r.URL, c.allowedHosts) { + origin, ok := baselineRequestOrigin(r.URL) + if !ok || (c.origin != "" && origin != c.origin) || (len(c.allowedHosts) > 0 && !baselineOriginAllowed(r.URL, c.allowedHosts)) { return false } return r.Method == "POST" && c.runtimeRequestTarget(r, "sessions") && baselineExactAPIVersionQuery(r.URL.RawQuery) diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 6476a1c..d0e66a4 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -260,6 +260,57 @@ func TestBaselineSessionOpenRejectsDuplicateMarkedPOSTBeforeInner(t *testing.T) } } +func TestBaselineSessionOpenOriginBindsToBeforeSnapshot(t *testing.T) { + const ( + beforeOrigin = "https://api.example:443" + path = "/tenant/v2/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview" + body = `{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"g01-test"}` + ) + for _, tc := range []struct { + name string + target string + reject bool + }{ + {name: "same origin", target: beforeOrigin + path}, + {name: "different approved origin", target: "https://other.example" + path, reject: true}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "session-open", + setID: 7, + organization: "fixture-org", + owner: "g01-test", + origin: beforeOrigin, + runtimePathPrefix: "/tenant/v2", + runtimePathPrefixSet: true, + allowedHosts: []string{"api.example", "other.example"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, tc.target, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + _, err = transport.RoundTrip(req) + if tc.reject { + if !errors.Is(err, ErrRemote) || innerCalls != 0 { + t.Fatalf("different approved session-open origin err=%v inner calls=%d, want remote rejection before inner transport", err, innerCalls) + } + if capture.requestObserved() { + t.Fatal("different approved session-open origin was marked observed") + } + return + } + if err != nil || innerCalls != 1 || !capture.requestObserved() { + t.Fatalf("same-origin session-open err=%v inner calls=%d observed=%v, want one forwarded request", err, innerCalls, capture.requestObserved()) + } + }) + } +} + func TestBaselineSnapshotRequestsRequireExactOriginBeforeInner(t *testing.T) { for _, tc := range []struct { name string diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index fcece25..0037746 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -391,10 +391,11 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook return nil, ErrRemote } hook.mu.Lock() + expectedOrigin := hook.origin expectedRuntimePathPrefix := hook.runtimePathPrefix expectedRuntimePathPrefixSet := hook.runtimePathPrefixSet hook.mu.Unlock() - wire := &baselineWireCapture{stage: "session-open", setID: id, organization: configured.approval.Organization, owner: owner, runtimePathPrefix: expectedRuntimePathPrefix, runtimePathPrefixSet: expectedRuntimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} + wire := &baselineWireCapture{stage: "session-open", setID: id, organization: configured.approval.Organization, owner: owner, origin: expectedOrigin, runtimePathPrefix: expectedRuntimePathPrefix, runtimePathPrefixSet: expectedRuntimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(configured.approval, configured.drainEndpointHost())} session, err := configured.client.MessageSessionClient(wire.context(c), id, owner, configured.options...) if err != nil { return nil, ErrRemote diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index fa4c3d3..9d0cdc5 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -457,6 +457,9 @@ func TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener(t *testing.T) if got := fixture.polls.Load(); got != 0 { t.Fatalf("session-open origin mismatch started listener polls: polls=%d", got) } + if got := fixture.sessionOpens.Load(); got != 1 { + t.Fatalf("session-open origin mismatch reached fixture: opens=%d, want setup open only", got) + } for _, event := range j.Events() { if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { t.Fatal("session-open origin mismatch produced an observed drain") From 2c3227780a5b5ef1ae4424ecbc50b53b795fbf96 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 05:25:53 +0900 Subject: [PATCH 45/63] docs(g01): record session origin finding evidence --- docs/evidence/g01-idle-drain.md | 85 +++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 8b45ac9..233f4bd 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1871,3 +1871,88 @@ and this evidence append is documentation-only. Rollback is recoverable with append can be reverted separately if required. No live rollback was run, and the unresolved live G01 gate, exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: marked session-open origin binding + +Date: 2026-09-13. This correction started from exact head +`91aa852cc2f66102aad54c402b7da8e552163828`. The fresh Codex P1 finding is +[marked session-open origin mismatch forwarded before rejection](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997399947). +No Project, Issue, goal, dependency or status field was changed. + +#### Red-first reproduction + +After adding the regression and before changing production code, this focused +command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestBaselineSessionOpenOriginBindsToBeforeSnapshot$' -count=1 -v -timeout=60s +``` + +The same-origin control passed, while the different-but-approved origin failed +with `err= inner calls=1`, proving the marked POST reached the inner +transport before the fix. The regression retained only fixed counters and +error categories; no request body, token, URL, response error or private log +was recorded. + +#### Correction and verification + +The before Scale Set snapshot origin now initializes the paired listener's +session-open capture, and the pinned drain session capture inherits the +before-snapshot origin from its poll hook. A marked session-open target now +rejects a different origin before body handling or inner transport forwarding; +same-origin session-open, the explicit bootstrap allowlist, unmarked requests, +and the existing tenant-prefix, Host and Opaque checks remain intact. The +correction source/test commit is +`3be170f82e4b173baf808f22e3637dfd533fe7d4`. + +The focused normal run exited 0 in 0.513s, including the new regression, the +pinned-SDK mismatch regression, same-origin session-open, bootstrap, +tenant-prefix, Opaque, Host and unmarked-forwarding controls: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineSessionOpenOriginBindsToBeforeSnapshot|TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener|TestBaselineSessionOpenTargetMismatchStopsBeforeInner|TestBaselineSessionOpenBodyMustMatchOwnerBeforeInner|TestBaselineRuntimePathPrefixMismatchStopsBeforeInner|TestMarkedRequestOpaqueStopsBeforeInner|TestMarkedSnapshotRewriteAllowsOnlyRequiredBootstrap|TestUnmarkedOpaqueRequestPreservesInnerForwarding|TestDrainPollHookForwardsUnmarkedNonPollRequest)$' -count=1 -v -timeout=240s +``` + +The focused race run exited 0 in 4.737s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestDrain|TestPinnedSDKDrain|TestDriverDrainThroughPinnedSDKAndPollHook|TestBaseline.*(Acquire|SessionOpen|Snapshot)|TestMarked|TestUnmarked)' -count=1 -timeout=300s +``` + +The full package normal run exited 0 in 28.681s and the full race run exited 0 +in 40.923s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=300s +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`bash scripts/gofmt.sh check`, and `git diff --check` each exited 0. The +diff secret/private-path scan exited 0 and printed +`diff secret/private-path scan passed`: + +```text +set -e +if git diff --text | rg -n '(/Users/|/home/|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})'; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The repository offline gate exited 0 and printed +`offline experiment checks passed: 2 module(s)`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +The exact tested implementation head is +`3be170f82e4b173baf808f22e3637dfd533fe7d4`; this evidence append is a separate +documentation-only commit. Rollback is recoverable with +`git revert --no-edit 3be170f82e4b173baf808f22e3637dfd533fe7d4`; no live +rollback, App, runner, workflow, canary, Docker/Lima, Keychain, launchd, +credential or cleanup operation was performed. The unresolved live G01 gate, +independent exact-head Codex review, CI and merge remain coordinator-owned. From e4e2a28fd24c767db51a83869369631e86b3c919 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 05:51:36 +0900 Subject: [PATCH 46/63] fix(g01): reject case-folded duplicate poll capacity --- experiments/g01-scaleset/livecanary/drain.go | 12 ++++++++++- .../livecanary/drain_p1_followup_test.go | 20 +++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 3b406ce..144e670 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -457,6 +457,16 @@ func sameDrainQuery(left, right url.Values) bool { return true } +func drainHeaderValues(header http.Header, name string) []string { + var values []string + for key, candidates := range header { + if strings.EqualFold(key, name) { + values = append(values, candidates...) + } + } + return values +} + func (h *drainPollHook) pollRequestValid(req *http.Request, attempt int) bool { if h == nil || req == nil || req.URL == nil || (attempt != 1 && attempt != 2) { return false @@ -490,7 +500,7 @@ func (h *drainPollHook) pollRequestValid(req *http.Request, attempt int) bool { if strings.EqualFold(want.Scheme, "https") && !runtimePathPrefixSet { return false } - values := req.Header.Values(scaleset.HeaderScaleSetMaxCapacity) + values := drainHeaderValues(req.Header, scaleset.HeaderScaleSetMaxCapacity) wantCapacity := drainInitialCapacity if attempt == 2 { wantCapacity = drainWithdrawnCapacity diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index d0e66a4..74f8bb4 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -1158,6 +1158,26 @@ func TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner(t *testing.T) { } } +func TestPinnedSDKDrainRejectsCaseFoldedDuplicateCapacityBeforeFixture(t *testing.T) { + a := approval() + fixture, _, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodGet && req.URL.Path == "/queue" && req.URL.Query().Get("lastMessageId") != "" { + // Direct map assignment models an intervening wrapper that bypasses + // Header.Set's canonicalization and adds a duplicate wire key. + req.Header[strings.ToLower(scaleset.HeaderScaleSetMaxCapacity)] = []string{"1"} + } + }, + }) + observation, err := runDrainListener(context.Background(), session, 7, hook) + if !errors.Is(err, ErrQuarantine) || observation.Outcome == drainOutcomeObserved { + t.Fatalf("case-folded duplicate capacity was promoted: observation=%+v err=%v", observation, err) + } + if got := fixture.polls.Load(); got != 1 { + t.Fatalf("case-folded duplicate capacity reached fixture: polls=%d, want first poll only", got) + } +} + func TestDrainCancellationStopsBeforeReleasingHeldResponse(t *testing.T) { var order []string cancelAndJoinDrain( From dff2e031707377d9d16a1ad39a4069a596abb1b9 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 05:53:06 +0900 Subject: [PATCH 47/63] docs(g01): record duplicate poll capacity evidence --- docs/evidence/g01-idle-drain.md | 88 +++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 233f4bd..1ff6cfc 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -1956,3 +1956,91 @@ documentation-only commit. Rollback is recoverable with rollback, App, runner, workflow, canary, Docker/Lima, Keychain, launchd, credential or cleanup operation was performed. The unresolved live G01 gate, independent exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: case-folded duplicate poll capacity header + +Date: 2026-09-13. This correction started from exact head +`2c3227780a5b5ef1ae4424ecbc50b53b795fbf96` and addresses the fresh Codex P1 +[case-folded duplicate marked poll capacity finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997489022). +The pinned SDK v0.4.0 header is `X-ScaleSetMaxCapacity`; no review request, +merge, Project/Issue/goal/status change, live App, runner, workflow, GitHub +API, credential, cleanup, or canary operation was performed. + +#### Red-first reproduction + +After adding `TestPinnedSDKDrainRejectsCaseFoldedDuplicateCapacityBeforeFixture` +and before changing the source boundary, this focused pinned-SDK TLS loopback +command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrainRejectsCaseFoldedDuplicateCapacityBeforeFixture$' -count=1 -v -timeout=120s +``` + +The canonical-only check accepted the lower-case map-key duplicate injected by +the intervening wrapper; the fixture saw both polls, the observation was +`Outcome:observed`, and the returned error was `err=`. The regression keeps +only fixed outcome/counter evidence and does not retain a request body, token, +URL, response error, or private log. + +#### Minimal correction and green evidence + +The marked poll boundary now scans every `http.Header` map key with +`strings.EqualFold`, collects all matching values, and requires exactly one +value equal to the intended capacity (`1` on the first poll and `0` on the +withdrawn poll). This rejects case-folded duplicate keys before the inner +transport while preserving the pinned SDK's valid canonical header, exact +origin/tenant/Host/Opaque/session/ACK/acquisition/close guards, unmarked +forwarding, and the existing snapshot bootstrap allowlist. + +The focused normal command exited 0 in 0.464s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrainRejectsCaseFoldedDuplicateCapacityBeforeFixture|TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner|TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects|TestPinnedSDKDrainAcquisitionRequiresStrictWireResponse|TestDrainPollHookRejectsDuplicatePhysicalWrites|TestDrainPollHookRequiresOneSuccessfulWritePerPoll|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=300s +``` + +The corresponding focused race command exited 0 in 1.962s with no race +diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrainRejectsCaseFoldedDuplicateCapacityBeforeFixture|TestPinnedSDKDrainRejectsPhysicalPollMutationBeforeInner|TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects|TestPinnedSDKDrainAcquisitionRequiresStrictWireResponse|TestDrainPollHookRejectsDuplicatePhysicalWrites|TestDrainPollHookRequiresOneSuccessfulWritePerPoll|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=300s +``` + +The complete offline `livecanary` package normal run exited 0 in 28.402s and +the race run exited 0 in 38.714s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=360s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=420s +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`GOTOOLCHAIN=go1.26.8 bash ../../scripts/gofmt.sh check`, and `git diff --check` +each exited 0. The final diff secret/private-path scan exited 0 and printed +`diff secret/private-path scan passed`: + +```text +set -e +path_pattern="$(printf '/%s/|/%s/' Users home)" +if git diff --text | rg -n "(${path_pattern}|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})"; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The two-module offline gate exited 0 and printed +`offline experiment checks passed: 2 module(s)`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +The exact tested source/evidence implementation head is +`e4e2a28fd24c767db51a83869369631e86b3c919` (`fix(g01): reject case-folded +duplicate poll capacity`); this evidence append is documentation-only on top +of that source commit. Rollback is recoverable with +`git revert --no-edit e4e2a28fd24c767db51a83869369631e86b3c919`; revert this +documentation append separately if needed. No live rollback was run, and the +unresolved live G01 gate, exact-head Codex review, CI and merge remain +coordinator-owned. From c68e8c2783a344a1863e873612dfa3336da3ad3c Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 06:24:39 +0900 Subject: [PATCH 48/63] fix(g01): bind terminal set origin --- .../livecanary/baseline_terminal.go | 10 ++- .../g01-scaleset/livecanary/baseline_wire.go | 9 +- .../livecanary/drain_p1_followup_test.go | 84 +++++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_terminal.go b/experiments/g01-scaleset/livecanary/baseline_terminal.go index ee0f846..b568d77 100644 --- a/experiments/g01-scaleset/livecanary/baseline_terminal.go +++ b/experiments/g01-scaleset/livecanary/baseline_terminal.go @@ -131,11 +131,12 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { r.Terminal.Roster = &observation known = e == nil && state.rosterMatches(&observation, func(ref controllerRecordRef) *Event { return s.event(workerRef(ref)) }) case stage == "terminal-set" || stage == "terminal-set-recheck" || stage == "terminal-set-absence": + origin := s.listener.capturedOrigin() runtimePathPrefix, runtimePathPrefixSet := s.listener.capturedRuntimePathPrefix() - if !runtimePathPrefixSet { + if origin == "" || !runtimePathPrefixSet { return ErrQuarantine } - capture := &baselineWireCapture{stage: stage, setID: s.setID, organization: s.approval.Organization, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} + capture := &baselineWireCapture{stage: stage, setID: s.setID, organization: s.approval.Organization, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) set, e := s.captured.GetScaleSet(capture.context(ctx), s.setID) callErr = e @@ -176,11 +177,12 @@ func (s *pairedBaselineScope) terminalStepCall(stage string) error { } known = e == nil && r.Terminal.Deletion != nil && receipt.AbsenceResult != nil case stage == "terminal-set-delete": + origin := s.listener.capturedOrigin() runtimePathPrefix, runtimePathPrefixSet := s.listener.capturedRuntimePathPrefix() - if !runtimePathPrefixSet { + if origin == "" || !runtimePathPrefixSet { return ErrQuarantine } - capture := &baselineWireCapture{stage: stage, setID: s.setID, organization: s.approval.Organization, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} + capture := &baselineWireCapture{stage: stage, setID: s.setID, organization: s.approval.Organization, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(s.approval, s.captured.drainEndpointHost())} ctx, cancel := context.WithTimeout(s.ctx, operationTimeout) callErr = s.captured.DeleteScaleSet(capture.context(ctx), s.setID) cancel() diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index bcbf7c9..4b7ce85 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -250,7 +250,14 @@ func (c *baselineWireCapture) snapshotOriginAllowed(r *http.Request) bool { if !ok { return false } - return c.origin == "" || c.origin == origin + if c.origin == "" { + // The initial unbound snapshot is allowed to establish the origin. Every + // terminal Scale Set snapshot is a post-session operation and must carry + // the listener's already captured origin instead of bootstrapping one + // from the request being authorized. + return !strings.HasPrefix(c.stage, "terminal-set") + } + return c.origin == origin } func snapshotRequestCandidate(r *http.Request) bool { diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 74f8bb4..1e3a9d0 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -348,6 +348,90 @@ func TestBaselineSnapshotRequestsRequireExactOriginBeforeInner(t *testing.T) { } } +func TestBaselineTerminalSetEffectsRequireCapturedOriginBeforeInner(t *testing.T) { + const ( + capturedOrigin = "https://api.example:443" + prefix = "/tenant/v2" + ) + for _, tc := range []struct { + name string + stage string + method string + status int + captureOrigin string + targetOrigin string + wantInner bool + }{ + { + name: "same-origin set deletion", + stage: "terminal-set-delete", + method: http.MethodDelete, + status: http.StatusNoContent, + captureOrigin: capturedOrigin, + targetOrigin: "https://api.example", + wantInner: true, + }, + { + name: "rewritten set deletion", + stage: "terminal-set-delete", + method: http.MethodDelete, + status: http.StatusNoContent, + targetOrigin: "https://other.example", + }, + { + name: "same-origin absence", + stage: "terminal-set-absence", + method: http.MethodGet, + status: http.StatusNotFound, + captureOrigin: capturedOrigin, + targetOrigin: "https://api.example", + wantInner: true, + }, + { + name: "rewritten absence", + stage: "terminal-set-absence", + method: http.MethodGet, + status: http.StatusNotFound, + targetOrigin: "https://other.example", + }, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: tc.stage, setID: 7, origin: tc.captureOrigin, + runtimePathPrefix: prefix, runtimePathPrefixSet: true, + allowedHosts: []string{"api.example", "other.example"}, + } + innerCalls := 0 + transport := responseBudgetTransport{inner: baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: tc.status, Body: http.NoBody, Request: req}, nil + })}} + target := tc.targetOrigin + prefix + "/_apis/runtime/runnerscalesets/7?api-version=6.0-preview" + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, target, nil) + if err != nil { + t.Fatal(err) + } + response, err := transport.RoundTrip(req) + if tc.wantInner { + if err != nil || innerCalls != 1 || response == nil || response.StatusCode != tc.status || !capture.requestObserved() { + t.Fatalf("same-origin %s err=%v inner calls=%d status=%v observed=%v, want one accepted request", tc.stage, err, innerCalls, responseStatus(response), capture.requestObserved()) + } + return + } + if !errors.Is(err, ErrRemote) || innerCalls != 0 || response != nil || capture.requestObserved() { + t.Fatalf("rewritten marked %s request was accepted before inner: err=%v inner calls=%d response=%v observed=%v", tc.stage, err, innerCalls, responseStatus(response), capture.requestObserved()) + } + }) + } +} + +func responseStatus(response *http.Response) any { + if response == nil { + return nil + } + return response.StatusCode +} + func TestBaselineRuntimePathPrefixMismatchStopsBeforeInner(t *testing.T) { const ( origin = "https://api.example:443" From e68d51f335f589d23ca49646489f4d3684072a77 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 06:26:09 +0900 Subject: [PATCH 49/63] docs(g01): record terminal origin evidence --- docs/evidence/g01-idle-drain.md | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 1ff6cfc..872b599 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2044,3 +2044,94 @@ of that source commit. Rollback is recoverable with documentation append separately if needed. No live rollback was run, and the unresolved live G01 gate, exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: terminal Scale Set deletion/absence origin binding + +Date: 2026-09-13. This correction started from exact head +`dff2e031707377d9d16a1ad39a4069a596abb1b9` and addresses the fresh Codex P1 +[terminal-set-delete origin capture finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997562319). +The marked terminal Scale Set DELETE capture previously left its origin empty; +the same omission affected the terminal absence GET. A request rewritten to a +different approved Actions origin could therefore reach the inner transport, +and a 404 from that origin could be accepted as the expected absence. No +Project, Issue, goal, dependency or status field was changed. + +#### Red-first reproduction + +The regression was added before the source correction and this focused command +exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestBaselineTerminalSetEffectsRequireCapturedOriginBeforeInner$' -count=1 -v -timeout=120s +``` + +The same-origin deletion and absence controls passed, while the rewritten +marked deletion and rewritten marked absence both reported `err=`, one +inner call, the rewritten request as observed, and statuses 204/404. This is +the intended boundary red: the different origin was approved by the host +allowlist but was not bound to the original listener origin. The test retains +only fixed counters/status categories; no request body, token, URL, response +error or private log is recorded. + +#### Minimal correction and green evidence + +Terminal Scale Set captures now copy the listener's captured session/snapshot +origin into every `terminal-set`, `terminal-set-recheck`, +`terminal-set-delete`, and `terminal-set-absence` capture. Marked terminal +Scale Set requests reject an empty captured origin and require exact canonical +origin equality before the inner transport or response validation; the +initial unbound snapshot remains allowed to establish its origin. The existing +tenant-prefix, Host/Opaque/session/ACK/acquisition/close/poll guards, +bootstrap allowlist, unmarked forwarding and no-live-cleanup contract remain +unchanged. + +The focused normal command exited 0 in 0.321s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineTerminalSetEffectsRequireCapturedOriginBeforeInner|TestBaselineSnapshotRequestsRequireExactOriginBeforeInner|TestBaselineSessionOpenOriginBindsToBeforeSnapshot|TestBaselineSessionCloseTargetRequiresExactOrigin|TestBaselineRuntimePathPrefixMismatchStopsBeforeInner)$' -count=1 -v -timeout=180s +``` + +The corresponding focused race command exited 0 in 1.498s with no race +diagnostics. The same-origin deletion and 404 absence controls each forwarded +exactly once; rewritten requests were rejected before the inner transport. + +The relevant pinned-SDK/TLS-loopback terminal integration controls exited 0 +(normal 1.980s, race 9.113s): + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags g01_pair_fixture ./livecanary -run '^TestPairedTerminal(ActualJournalsFinalize|ExportedAdapterActualJournalsFinalize)$' -count=1 -v -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags g01_pair_fixture -race ./livecanary -run '^TestPairedTerminal(ActualJournalsFinalize|ExportedAdapterActualJournalsFinalize)$' -count=1 -timeout=360s +``` + +The full `livecanary` package normal run exited 0 in 27.555s and the race run +exited 0 in 40.396s, with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=360s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=420s +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`GOTOOLCHAIN=go1.26.8 bash ../../scripts/gofmt.sh check`, and `git diff --check` +each exited 0. The diff secret/private-path scan exited 0 and printed +`diff secret/private-path scan passed`; the two-module offline gate exited 0 +and printed `offline experiment checks passed: 2 module(s)`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +The exact tested source head is +`c68e8c2783a344a1863e873612dfa3336da3ad3c` (`fix(g01): bind terminal set +origin`); the final source/evidence head is this source commit plus the +documentation-only append. Rollback is recoverable with +`git revert --no-edit c68e8c2783a344a1863e873612dfa3336da3ad3c`; revert this +documentation append separately if needed. No live rollback, App, runner, +workflow, canary, Docker/Lima, Keychain, launchd, credential or cleanup +operation was performed, and no live App/runner/canary evidence is claimed. +The unresolved live G01 gate, independent exact-head Codex review, CI and +merge remain coordinator-owned. From cbab4d80d44ddbd44dc4226bafa8e626fe1c119d Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 07:19:21 +0900 Subject: [PATCH 50/63] fix(g01): bind drain wire close and session auth --- .../livecanary/baseline_message.go | 11 +- .../g01-scaleset/livecanary/baseline_wire.go | 43 +++++++ experiments/g01-scaleset/livecanary/drain.go | 51 ++++++-- .../g01-scaleset/livecanary/drain_driver.go | 10 +- .../livecanary/drain_p1_followup_test.go | 69 +++++++++++ experiments/g01-scaleset/livecanary/sdk.go | 7 +- .../livecanary/sdk_integration_test.go | 116 ++++++++++++++++-- .../g01-scaleset/livecanary/sdk_test.go | 38 ++++++ 8 files changed, 321 insertions(+), 24 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_message.go b/experiments/g01-scaleset/livecanary/baseline_message.go index c441b8c..4bc6c4d 100644 --- a/experiments/g01-scaleset/livecanary/baseline_message.go +++ b/experiments/g01-scaleset/livecanary/baseline_message.go @@ -57,6 +57,7 @@ type baselineSessionFacts struct { SetName string `json:"set_name"` GroupID int `json:"group_id"` queueURL string `json:"-"` // ephemeral drain validation only; never serialized + authorization string `json:"-"` // ephemeral drain validation only; never serialized } // baselineRunnerFacts retains only the bounded fields needed to compare the @@ -202,6 +203,14 @@ func baselineText(s string, limit int) bool { } return true } + +// validDrainAuthorizationToken accepts only a bounded, header-safe opaque +// token. The value is retained solely in memory to corroborate the pinned SDK +// session and bind later marked runtime requests. +func validDrainAuthorizationToken(value string) bool { + return value != "" && baselineText(value, 4096) && strings.TrimSpace(value) == value && !strings.ContainsAny(value, "\r\n\x00") +} + func baselineStats(data json.RawMessage) (*baselineStatistics, error) { if len(data) == 0 || string(data) == "null" { return nil, nil @@ -363,7 +372,7 @@ func decodeBaselineSession(data []byte) (*baselineSessionFacts, error) { if err != nil { return nil, err } - x := &baselineSessionFacts{SessionID: w.ID, Owner: w.Owner, Statistics: s, queueURL: w.URL} + x := &baselineSessionFacts{SessionID: w.ID, Owner: w.Owner, Statistics: s, queueURL: w.URL, authorization: w.Token} if len(w.Set) > 0 && string(w.Set) != "null" { // Set contains SDK-defined fields not used here; ambiguity is rejected, // but unrelated forward-compatible set metadata is not copied to history. diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 4b7ce85..d9534bf 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -26,6 +26,8 @@ type baselineWireCapture struct { runnerName string sessionID string queue string // private, captured from the exact session; never journaled + authorization string // private, captured from the exact session; never journaled + requestAuthorization string // private, captured from the session-open request; never journaled runtimePathPrefix string // private tenant path prefix; never journaled runtimePathPrefixSet bool allowedHosts []string @@ -349,6 +351,34 @@ func baselineWireAllowedHosts(a Approval, apiHost string) []string { return hosts } +func exactDrainAuthorization(header http.Header, token string) bool { + got, ok := drainAuthorizationToken(header) + return ok && got == token +} + +func drainAuthorizationToken(header http.Header) (string, bool) { + values := drainHeaderValues(header, "Authorization") + if len(values) != 1 || !strings.HasPrefix(values[0], "Bearer ") { + return "", false + } + token := strings.TrimPrefix(values[0], "Bearer ") + if !validDrainAuthorizationToken(token) { + return "", false + } + return token, true +} + +// authorizationMatches is intentionally optional for synthetic state-machine +// transports that have no opened pinned SDK session. Production marked drain +// captures always receive the non-empty token from OpenDrainSession and thus +// require one exact Authorization header before forwarding. +func (c *baselineWireCapture) authorizationMatches(req *http.Request) bool { + if c == nil || c.authorization == "" { + return true + } + return req != nil && exactDrainAuthorization(req.Header, c.authorization) +} + // baselineRequestCaptureTransport runs immediately above the physical // transport. User-supplied test wrappers may mutate a request before it gets // here, so this is the final request-side boundary before any bytes leave the @@ -409,6 +439,9 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { if c == nil { return nil } + if (c.stage == "ack" || c.stage == "acquire" || c.stage == "terminal-session-close") && !c.authorizationMatches(req) { + return c.rejectRequest(req) + } if c.stage == "ack" { if !c.target(req) { return c.rejectRequest(req) @@ -437,6 +470,7 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { return c.rejectRequest(req) } c.requestCount = 1 + c.requestAuthorization, _ = drainAuthorizationToken(req.Header) c.mu.Unlock() data, err := readBaselineRequestBody(req.Body) req.Body = nil @@ -636,6 +670,15 @@ func (c *baselineWireCapture) requestObserved() bool { return c.requestCount == 1 && !c.invalid } +func (c *baselineWireCapture) requestAuthorizationValue() (string, bool) { + if c == nil { + return "", false + } + c.mu.Lock() + defer c.mu.Unlock() + return c.requestAuthorization, validDrainAuthorizationToken(c.requestAuthorization) && !c.invalid +} + // Run after the ordinary response budget has wrapped the body. Only this // explicitly marked exact request is read ahead; the SDK receives identical // bytes. Nothing is captured from bootstrap, unrelated or unmarked traffic. diff --git a/experiments/g01-scaleset/livecanary/drain.go b/experiments/g01-scaleset/livecanary/drain.go index 144e670..60cd740 100644 --- a/experiments/g01-scaleset/livecanary/drain.go +++ b/experiments/g01-scaleset/livecanary/drain.go @@ -335,6 +335,8 @@ type drainPollHook struct { mu sync.Mutex runtimePathPrefix string runtimePathPrefixSet bool + authorization string // private session token; never journaled + closeAuthorization string // private opened-session admin token; never journaled wroteRequest bool withdrawalCompleted bool responseBeforeWrite bool @@ -473,6 +475,7 @@ func (h *drainPollHook) pollRequestValid(req *http.Request, attempt int) bool { } h.mu.Lock() target := h.target + authorization := h.authorization messageID := 0 if attempt == 2 && h.batches[1] != nil { messageID = h.batches[1].MessageID @@ -500,6 +503,9 @@ func (h *drainPollHook) pollRequestValid(req *http.Request, attempt int) bool { if strings.EqualFold(want.Scheme, "https") && !runtimePathPrefixSet { return false } + if authorization != "" && !exactDrainAuthorization(req.Header, authorization) { + return false + } values := drainHeaderValues(req.Header, scaleset.HeaderScaleSetMaxCapacity) wantCapacity := drainInitialCapacity if attempt == 2 { @@ -738,14 +744,17 @@ func (h *drainPollHook) pollMessageState(index int) (present, absent bool) { // strict embedded job facts. It never persists, logs or rewrites the response // payload. type drainObservedBody struct { - source io.ReadCloser - status int - data []byte - over bool - mu sync.Mutex - finished bool - readFailed bool - onComplete func(drainStatistics, bool, *baselineBatch, bool, bool, bool) + source io.ReadCloser + status int + data []byte + over bool + mu sync.Mutex + finishMu sync.Mutex + finished bool + invalidated bool + readFailed bool + closeFailed bool + onComplete func(drainStatistics, bool, *baselineBatch, bool, bool, bool) } func (b *drainObservedBody) capture(data []byte) { @@ -765,16 +774,30 @@ func (b *drainObservedBody) capture(data []byte) { } func (b *drainObservedBody) finish() { + b.finishMu.Lock() + defer b.finishMu.Unlock() + b.mu.Lock() if b.finished { + invalidate := b.closeFailed && !b.invalidated + if invalidate { + b.invalidated = true + } b.mu.Unlock() + if invalidate && b.onComplete != nil { + b.onComplete(drainStatistics{}, false, nil, false, false, false) + } return } b.finished = true data := append([]byte(nil), b.data...) over := b.over readFailed := b.readFailed + closeFailed := b.closeFailed b.data = nil + if over || readFailed || closeFailed { + b.invalidated = true + } b.mu.Unlock() stats, known := drainStatisticsFromBody(data) @@ -789,7 +812,7 @@ func (b *drainObservedBody) finish() { present = false absent = false } - if over || readFailed { + if over || readFailed || closeFailed { known = false stats = drainStatistics{} batch = nil @@ -844,9 +867,15 @@ func (b *drainObservedBody) Close() error { b.over = true b.mu.Unlock() } - b.finish() } - return b.source.Close() + closeErr := b.source.Close() + if closeErr != nil { + b.mu.Lock() + b.closeFailed = true + b.mu.Unlock() + } + b.finish() + return closeErr } func (h *drainPollHook) releaseResponse() { diff --git a/experiments/g01-scaleset/livecanary/drain_driver.go b/experiments/g01-scaleset/livecanary/drain_driver.go index 6ff076d..7be6ebb 100644 --- a/experiments/g01-scaleset/livecanary/drain_driver.go +++ b/experiments/g01-scaleset/livecanary/drain_driver.go @@ -133,6 +133,7 @@ func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error c.hook.mu.Lock() queue := c.hook.target origin := c.hook.origin + authorization := c.hook.authorization runtimePathPrefix := c.hook.runtimePathPrefix runtimePathPrefixSet := c.hook.runtimePathPrefixSet c.hook.mu.Unlock() @@ -160,7 +161,7 @@ func (c *journaledDrainClient) DeleteMessage(ctx context.Context, id int) error if c.d != nil { approval = c.d.Approval } - wire = &baselineWireCapture{stage: "ack", setID: setID, sessionID: sessionID, queue: queue, cursor: id, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(approval, apiHost)} + wire = &baselineWireCapture{stage: "ack", setID: setID, sessionID: sessionID, queue: queue, cursor: id, origin: origin, authorization: authorization, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(approval, apiHost)} call = wire.context(call) } if err := c.inner.DeleteMessage(call, id); err != nil { @@ -195,6 +196,7 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] } c.hook.mu.Lock() queue, origin := c.hook.target, c.hook.origin + authorization := c.hook.authorization runtimePathPrefix := c.hook.runtimePathPrefix runtimePathPrefixSet := c.hook.runtimePathPrefixSet c.hook.mu.Unlock() @@ -205,7 +207,7 @@ func (c *journaledDrainClient) AcquireJobs(ctx context.Context, ids []int64) ([] if reader, ok := c.d.API.(drainEndpointHostReader); ok { apiHost = reader.drainEndpointHost() } - wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, requestIDs: slices.Clone(ids), origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} + wire = &baselineWireCapture{stage: "acquire", setID: setID, queue: queue, requestIDs: slices.Clone(ids), origin: origin, authorization: authorization, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: baselineWireAllowedHosts(c.d.Approval, apiHost)} call = wire.context(call) } got, err = c.inner.AcquireJobs(call, slices.Clone(ids)) @@ -451,11 +453,13 @@ func (d *Driver) drain(ctx context.Context, setID int) error { } closeErr := d.effect(ctx, "session-close", nil, func(call context.Context) (Event, error) { origin := "" + closeAuthorization := "" runtimePathPrefix := "" runtimePathPrefixSet := false if hook != nil { hook.mu.Lock() origin = hook.origin + closeAuthorization = hook.closeAuthorization runtimePathPrefix = hook.runtimePathPrefix runtimePathPrefixSet = hook.runtimePathPrefixSet hook.mu.Unlock() @@ -467,7 +471,7 @@ func (d *Driver) drain(ctx context.Context, setID int) error { if reader, ok := d.API.(drainEndpointHostReader); ok { allowedHosts = baselineWireAllowedHosts(d.Approval, reader.drainEndpointHost()) } - wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID, origin: origin, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: allowedHosts} + wire := &baselineWireCapture{stage: "terminal-session-close", setID: setID, sessionID: sessionID, origin: origin, authorization: closeAuthorization, runtimePathPrefix: runtimePathPrefix, runtimePathPrefixSet: runtimePathPrefixSet, allowedHosts: allowedHosts} call = wire.context(call) if err := session.Close(call); err != nil { return Event{}, err diff --git a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go index 1e3a9d0..9acfc26 100644 --- a/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go +++ b/experiments/g01-scaleset/livecanary/drain_p1_followup_test.go @@ -581,6 +581,75 @@ func TestMarkedPollOpaqueStopsBeforeInner(t *testing.T) { } } +func TestMarkedRuntimeAuthorizationMismatchStopsBeforeInner(t *testing.T) { + token := strings.Repeat("t", 24) + wrong := strings.Repeat("x", 24) + const ( + origin = "https://fixture.invalid:443" + prefix = "/tenant/v2" + ) + t.Run("poll", func(t *testing.T) { + const target = "https://fixture.invalid/queue?proof=fixture" + hook := newDrainPollHookWithOriginAndPrefix(target, origin, prefix) + hook.mu.Lock() + hook.authorization = token + hook.mu.Unlock() + innerCalls := 0 + hook.inner = drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusAccepted, Body: http.NoBody, Request: req}, nil + }) + req, err := http.NewRequestWithContext(hook.markPoll(context.Background()), http.MethodGet, target, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set(scaleset.HeaderScaleSetMaxCapacity, "1") + req.Header.Set("Authorization", "Bearer "+wrong) + if _, err := hook.RoundTrip(req); !errors.Is(err, ErrQuarantine) { + t.Fatalf("mismatched poll authorization = %v, want quarantine", err) + } + if innerCalls != 0 { + t.Fatalf("mismatched poll authorization reached inner transport: calls=%d", innerCalls) + } + }) + + for _, tc := range []struct { + name string + stage string + method string + target string + body string + }{ + {name: "ACK", stage: "ack", method: http.MethodDelete, target: "https://fixture.invalid/queue/41?proof=fixture"}, + {name: "acquisition", stage: "acquire", method: http.MethodPost, target: "https://fixture.invalid" + prefix + "/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", body: `[41]`}, + {name: "session close", stage: "terminal-session-close", method: http.MethodDelete, target: "https://fixture.invalid" + prefix + "/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview"}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: tc.stage, setID: 7, sessionID: "session", queue: "https://fixture.invalid/queue?proof=fixture", cursor: 41, + origin: origin, authorization: token, runtimePathPrefix: prefix, runtimePathPrefixSet: true, + requestIDs: []int64{41}, allowedHosts: []string{"fixture.invalid"}, + } + innerCalls := 0 + transport := baselineRequestCaptureTransport{inner: drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusNoContent, Body: http.NoBody, Request: req}, nil + })} + req, err := http.NewRequestWithContext(capture.context(context.Background()), tc.method, tc.target, strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Authorization", "Bearer "+wrong) + if _, err := transport.RoundTrip(req); !errors.Is(err, ErrRemote) { + t.Fatalf("mismatched %s authorization = %v, want remote rejection", tc.name, err) + } + if innerCalls != 0 { + t.Fatalf("mismatched %s authorization reached inner transport: calls=%d", tc.name, innerCalls) + } + }) + } +} + func TestMarkedSnapshotRewriteAllowsOnlyRequiredBootstrap(t *testing.T) { for _, stage := range []string{"set-observe", "runner-observe", "terminal-set"} { for _, tc := range []struct { diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index 0037746..cbd1d31 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -353,7 +353,7 @@ func drainApprovedHostPort(value string) (string, string, bool) { } func validDrainSessionWire(a Approval, id int, owner string, wire *baselineSessionFacts, session scaleset.RunnerScaleSetSession) bool { - if wire == nil || session.SessionID == [16]byte{} || wire.SessionID != session.SessionID.String() || wire.Owner != owner || session.OwnerName != owner || !validDrainQueueURL(wire.queueURL, a.ActionsHosts) || wire.queueURL != session.MessageQueueURL || session.MessageQueueAccessToken == "" || !wire.Statistics.completeDrain() || !wire.NestedSet || wire.SetID != id || wire.SetName != owner || wire.GroupID != a.RunnerGroupID || !wire.NestedStatistics.completeDrain() || !wire.Statistics.matches(session.Statistics) { + if wire == nil || session.SessionID == [16]byte{} || wire.SessionID != session.SessionID.String() || wire.Owner != owner || session.OwnerName != owner || !validDrainQueueURL(wire.queueURL, a.ActionsHosts) || wire.queueURL != session.MessageQueueURL || !validDrainAuthorizationToken(wire.authorization) || wire.authorization != session.MessageQueueAccessToken || !wire.Statistics.completeDrain() || !wire.NestedSet || wire.SetID != id || wire.SetName != owner || wire.GroupID != a.RunnerGroupID || !wire.NestedStatistics.completeDrain() || !wire.Statistics.matches(session.Statistics) { return false } set := session.RunnerScaleSet @@ -403,7 +403,8 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook sessionFacts, _, _, status := wire.facts() sessionOrigin := wire.requestOrigin() sessionRuntimePathPrefix, prefixKnown := wire.requestRuntimePathPrefix() - if session == nil || !wire.observed() || status != http.StatusOK || sessionOrigin == "" || !prefixKnown || !validDrainSessionWire(configured.approval, id, owner, sessionFacts, session.Session()) { + sessionAuthorization, authorizationKnown := wire.requestAuthorizationValue() + if session == nil || !wire.observed() || status != http.StatusOK || sessionOrigin == "" || !prefixKnown || !authorizationKnown || !validDrainSessionWire(configured.approval, id, owner, sessionFacts, session.Session()) { return nil, ErrQuarantine } hook.mu.Lock() @@ -416,6 +417,8 @@ func (a *SDKAPI) OpenDrainSession(c context.Context, id int, owner string, hook hook.origin = sessionOrigin hook.runtimePathPrefix = sessionRuntimePathPrefix hook.runtimePathPrefixSet = true + hook.authorization = sessionFacts.authorization + hook.closeAuthorization = sessionAuthorization hook.mu.Unlock() return session, nil } diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 9d0cdc5..27f10f7 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -657,6 +657,100 @@ func TestPinnedSDKDrainRejectsNonEOFPollReadError(t *testing.T) { } } +func TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects(t *testing.T) { + a := approval() + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{pollCloseError: io.ErrClosedPipe}) + hook.releaseResponse() + j := &memoryJournal{} + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + client := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), setID: 7, hook: hook} + + message, err := client.GetMessage(context.Background(), 0, drainInitialCapacity) + if err == nil || message != nil { + t.Fatalf("poll close error = message %v err %v, want rejection", message, err) + } + if _, known := hook.pollStatistics(1); known { + t.Fatal("poll close error published known statistics") + } + if _, known := hook.pollBatch(1); known { + t.Fatal("poll close error published a known batch") + } + if present, absent := hook.pollMessageState(1); present || absent { + t.Fatalf("poll close error published message state: present=%v absent=%v", present, absent) + } + if fixture.acks.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("poll close error reached effects: ack=%d acquire=%d", fixture.acks.Load(), fixture.acquires.Load()) + } + for _, event := range j.Events() { + if event.Kind == "result" && (event.Operation == "ack" || event.Operation == "acquire" || event.Operation == "drain") { + t.Fatalf("poll close error recorded an observed effect: operation=%s", event.Operation) + } + } +} + +func TestPinnedSDKDrainRejectsSubstitutedSessionAuthorizationBeforeEffects(t *testing.T) { + a := approval() + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodPost && strings.HasSuffix(req.URL.Path, "/acquirejobs") { + req.Header.Set("Authorization", "Bearer "+strings.Repeat("x", 24)) + } + }, + }) + hook.releaseResponse() + j := &memoryJournal{} + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + client := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), setID: 7, hook: hook} + message, err := client.GetMessage(context.Background(), 0, drainInitialCapacity) + if err != nil || message == nil { + t.Fatalf("valid setup poll = message %v err %v", message, err) + } + if err := client.DeleteMessage(context.Background(), message.MessageID); err != nil { + t.Fatalf("valid setup ACK = %v", err) + } + got, err := client.AcquireJobs(context.Background(), []int64{41}) + if !errors.Is(err, ErrQuarantine) || got != nil { + t.Fatalf("substituted session authorization = got %v err %v, want pre-inner quarantine", got, err) + } + if fixture.acquires.Load() != 0 { + t.Fatalf("substituted session authorization reached acquisition endpoint: acquires=%d", fixture.acquires.Load()) + } + if state := replay(j.Events()); !state.uncertain { + t.Fatal("substituted session authorization did not retain uncertainty") + } +} + +func TestPinnedSDKDrainRejectsSubstitutedSessionCloseAuthorizationBeforeEffects(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + fixture, base, _, _ := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ + mutateRequest: func(req *http.Request) { + if req.Method == http.MethodDelete && strings.Contains(req.URL.Path, "/sessions/") { + req.Header.Set("Authorization", "Bearer "+strings.Repeat("x", 24)) + } + }, + }) + j := &memoryJournal{} + for _, event := range drainReplayPrefix()[:3] { + if err := j.Append(event); err != nil { + t.Fatal(err) + } + } + d := &Driver{Approval: a, Journal: j, API: fixtureSDK{base}} + err := d.Run(context.Background(), "drain") + if !errors.Is(err, ErrQuarantine) { + t.Fatalf("substituted session-close authorization = %v, want quarantine", err) + } + if fixture.closeRequests.Load() != 0 { + t.Fatalf("substituted session-close authorization reached close endpoint: closes=%d", fixture.closeRequests.Load()) + } + for _, event := range j.Events() { + if event.Kind == "observation" && event.Operation == "drain" && event.Drain != nil && event.Drain.Outcome == drainOutcomeObserved { + t.Fatal("substituted session-close authorization produced an observed drain") + } + } +} + func TestPinnedSDKDrainBindsACKToPhysicalDelete(t *testing.T) { a := approval() fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ @@ -1047,6 +1141,7 @@ type pinnedDrainOptions struct { allowOpenError bool acceptAnyACK bool pollReadError error + pollCloseError error snapshotBodies []string mutateRequest func(*http.Request) wrongCloseOrigin bool @@ -1235,9 +1330,9 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options } wrappers := []func(http.RoundTripper) http.RoundTripper(nil) if hook != nil { - if options.pollReadError != nil { + if options.pollReadError != nil || options.pollCloseError != nil { wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { - return sdkResponseBodyFaultRoundTripper{inner: inner, path: "/queue", err: options.pollReadError} + return sdkResponseBodyFaultRoundTripper{inner: inner, path: "/queue", err: options.pollReadError, closeErr: options.pollCloseError} }) } wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { @@ -1323,9 +1418,10 @@ func (f *pinnedDrainFixture) acquireRequestBody() string { } type sdkResponseBodyFaultRoundTripper struct { - inner http.RoundTripper - path string - err error + inner http.RoundTripper + path string + err error + closeErr error } func (t sdkResponseBodyFaultRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { @@ -1333,13 +1429,14 @@ func (t sdkResponseBodyFaultRoundTripper) RoundTrip(req *http.Request) (*http.Re if err != nil || response == nil || req.URL == nil || req.URL.Path != t.path || response.Body == nil { return response, err } - response.Body = &sdkResponseBodyFault{source: response.Body, err: t.err} + response.Body = &sdkResponseBodyFault{source: response.Body, err: t.err, closeErr: t.closeErr} return response, nil } type sdkResponseBodyFault struct { source io.ReadCloser err error + closeErr error injected bool pending bool } @@ -1361,7 +1458,12 @@ func (b *sdkResponseBodyFault) Read(p []byte) (int, error) { return n, err } -func (b *sdkResponseBodyFault) Close() error { return b.source.Close() } +func (b *sdkResponseBodyFault) Close() error { + if err := b.source.Close(); err != nil { + return err + } + return b.closeErr +} type sdkRequestMutationRoundTripper struct { inner http.RoundTripper diff --git a/experiments/g01-scaleset/livecanary/sdk_test.go b/experiments/g01-scaleset/livecanary/sdk_test.go index 57f97eb..6229f6a 100644 --- a/experiments/g01-scaleset/livecanary/sdk_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_test.go @@ -8,6 +8,9 @@ import ( "strings" "testing" "time" + + "github.com/actions/scaleset" + "github.com/google/uuid" ) func credentials(a Approval) Credentials { @@ -106,6 +109,41 @@ func TestCredentialAttestationMismatchAndExpiredTokenRejected(t *testing.T) { } } +func TestValidDrainSessionWireRequiresExactQueueAuthorization(t *testing.T) { + a := approval() + token := strings.Repeat("q", 24) + other := strings.Repeat("w", 24) + zero, one := 0, 1 + wireStats := &baselineStatistics{Available: &zero, Acquired: &zero, Assigned: &zero, Running: &zero, Registered: &one, Busy: &zero, Idle: &one} + sdkStats := &scaleset.RunnerScaleSetStatistic{TotalAvailableJobs: 0, TotalAcquiredJobs: 0, TotalAssignedJobs: 0, TotalRunningJobs: 0, TotalRegisteredRunners: 1, TotalBusyRunners: 0, TotalIdleRunners: 1} + set := &scaleset.RunnerScaleSet{ + ID: 7, Name: a.setName(), RunnerGroupID: a.RunnerGroupID, + Labels: []scaleset.Label{{Name: a.setName()}}, + RunnerSetting: scaleset.RunnerSetting{DisableUpdate: true}, Statistics: sdkStats, + } + sessionID := uuid.MustParse("00000000-0000-4000-8000-000000000031") + session := scaleset.RunnerScaleSetSession{ + SessionID: sessionID, OwnerName: a.setName(), MessageQueueURL: "https://fixture.actions.githubusercontent.com/queue", + MessageQueueAccessToken: token, RunnerScaleSet: set, Statistics: sdkStats, + } + wire := &baselineSessionFacts{ + SessionID: sessionID.String(), Owner: a.setName(), Statistics: wireStats, NestedStatistics: wireStats, + NestedSet: true, SetID: 7, SetName: a.setName(), GroupID: a.RunnerGroupID, + queueURL: session.MessageQueueURL, authorization: token, + } + if !validDrainSessionWire(a, 7, a.setName(), wire, session) { + t.Fatal("matching queue authorization rejected") + } + wire.authorization = other + if validDrainSessionWire(a, 7, a.setName(), wire, session) { + t.Fatal("different queue authorization accepted") + } + wire.authorization = "" + if validDrainSessionWire(a, 7, a.setName(), wire, session) { + t.Fatal("missing queue authorization accepted") + } +} + func TestTransportRejectsPlaintextOffHostAndProxyBeforeNetwork(t *testing.T) { a := approval() api, err := NewSDKAPI(a, credentials(a)) From a8bfc277e7b663034dc1903bd9e8d758e3b0c88b Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 07:20:34 +0900 Subject: [PATCH 51/63] docs(g01): record drain wire P1 evidence --- docs/evidence/g01-idle-drain.md | 95 +++++++++++++++++++++++++++++++++ 1 file changed, 95 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 872b599..3e7a29d 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2135,3 +2135,98 @@ workflow, canary, Docker/Lima, Keychain, launchd, credential or cleanup operation was performed, and no live App/runner/canary evidence is claimed. The unresolved live G01 gate, independent exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: poll close failure and opened-session authorization binding + +Date: 2026-09-13. This correction started from exact head +`e68d51f335f589d23ca49646489f4d3684072a77` and addresses the fresh Codex P1 +[poll close failure finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997740103) +and [opened-session authorization finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997740106). +No Project, Issue, goal, dependency, status or branch field was changed; no +review request, live App/runner/canary operation, workflow replay, Docker/Lima, +Keychain, launchd, credential or cleanup operation was performed. + +#### Red-first reproduction + +The two regressions were added before the source correction. This focused +pinned-SDK/TLS-loopback command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrain(RejectsPollCloseErrorBeforeEffects|RejectsSubstitutedSessionAuthorizationBeforeEffects)$' -count=1 -v -timeout=180s +``` + +The close-failure regression reported `poll close error published known +statistics`; the authorization regression reported `got [41] err , want +pre-inner quarantine`. Thus complete poll bytes plus a failing `Close` could +publish known facts, and a nonempty substituted runtime authorization could +reach acquisition. The regressions retain only fixed outcome, status and +counter assertions; no raw token, Authorization value, response body, SDK +error, URL or private log is retained. + +#### Minimal correction and green evidence + +The bounded poll body now records the source `Close` error before finalizing an +unfinished body and synchronously retracts all statistics, batch and +message-presence facts if EOF had already published an initial snapshot; +clean EOF and existing non-EOF read-error behavior remain unchanged. Strict +session capture retains the message-queue token only in private memory, +compares it exactly with the pinned SDK session value, and captures the exact +opened-session admin authorization for the pinned SDK close request. Marked +poll, ACK, acquisition and session-close requests require one +case-insensitive-header-key match with one exact `Bearer` value before the +inner transport, while valid same-session traffic and existing route, origin, +tenant-prefix, Host/Opaque, target/cardinality, snapshot and unmarked controls +remain covered. + +The focused normal command exited 0 in 2.314s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsSubstitutedSessionAuthorizationBeforeEffects|TestPinnedSDKDrainRejectsSubstitutedSessionCloseAuthorizationBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestMarkedRuntimeAuthorizationMismatchStopsBeforeInner|TestValidDrainSessionWireRequiresExactQueueAuthorization|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=300s +``` + +The corresponding focused race command exited 0 in 4.067s with no race +diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsSubstitutedSessionAuthorizationBeforeEffects|TestPinnedSDKDrainRejectsSubstitutedSessionCloseAuthorizationBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestMarkedRuntimeAuthorizationMismatchStopsBeforeInner|TestValidDrainSessionWireRequiresExactQueueAuthorization|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=300s +``` + +The full `livecanary` package normal run exited 0 in 29.032s and the full race +run exited 0 in 39.918s, with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=360s +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check`, and `git diff --check` +each exited 0. The diff secret/private-path scan exited 0 and printed +`diff secret/private-path scan passed`: + +```text +set -e +path_pattern="$(printf '/%s/|/%s/' Users home)" +if git diff --text | rg -n "(${path_pattern}|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})"; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The two-module offline gate exited 0 and printed +`offline experiment checks passed: 2 module(s)`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +The exact tested implementation source head is +`cbab4d80d44ddbd44dc4226bafa8e626fe1c119d4` (`fix(g01): bind drain wire +close and session auth`); this evidence append is a separate documentation-only +commit. Rollback is recoverable with +`git revert --no-edit cbab4d80d44ddbd44dc4226bafa8e626fe1c119d4`; revert this +documentation append separately if needed. No live App/runner/canary evidence +is claimed; the unresolved live G01 gate, independent exact-head Codex review, +CI and merge remain coordinator-owned. From 3c8567795c69ec3d4b0171b685d0ac21506f83f5 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 08:15:23 +0900 Subject: [PATCH 52/63] fix(g01): fence marked wire requests across wrappers --- .../g01-scaleset/livecanary/baseline_wire.go | 47 +++- .../livecanary/baseline_wire_p1_test.go | 235 ++++++++++++++++++ .../livecanary/response_budget.go | 17 +- 3 files changed, 293 insertions(+), 6 deletions(-) create mode 100644 experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index d9534bf..d652cb8 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -379,6 +379,32 @@ func (c *baselineWireCapture) authorizationMatches(req *http.Request) bool { return req != nil && exactDrainAuthorization(req.Header, c.authorization) } +// baselineWireMarkerHeader is an ephemeral in-process transport marker. It is +// copied by the standard Request.Clone path used by SDK/intervening wrappers, +// then removed at the final request boundary before any physical transport is +// called. Its value carries no operation, credential or response data. +const baselineWireMarkerHeader = "X-GH-Runnerd-Baseline-Wire" +const baselineWireMarkerValue = "1" + +func baselineWireMarkerValues(req *http.Request) []string { + if req == nil || req.Header == nil { + return nil + } + return req.Header.Values(baselineWireMarkerHeader) +} + +func markBaselineWireRequest(req *http.Request) *http.Request { + if req == nil { + return nil + } + marked := req.Clone(req.Context()) + if marked.Header == nil { + marked.Header = make(http.Header) + } + marked.Header.Set(baselineWireMarkerHeader, baselineWireMarkerValue) + return marked +} + // baselineRequestCaptureTransport runs immediately above the physical // transport. User-supplied test wrappers may mutate a request before it gets // here, so this is the final request-side boundary before any bytes leave the @@ -393,8 +419,25 @@ func baselineRequestHostMatchesURL(req *http.Request) bool { } func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req == nil { + return nil, ErrRemote + } c, _ := req.Context().Value(baselineWireKey{}).(*baselineWireCapture) + markerValues := baselineWireMarkerValues(req) + if c == nil && len(markerValues) > 0 { + if req.Body != nil { + _ = req.Body.Close() + } + return nil, ErrRemote + } if c != nil { + if len(markerValues) > 0 { + if len(markerValues) != 1 || markerValues[0] != baselineWireMarkerValue { + return nil, c.rejectRequest(req) + } + // Never expose the private marker to the physical transport. + req.Header.Del(baselineWireMarkerHeader) + } if !baselineRequestHostMatchesURL(req) { return nil, c.rejectRequest(req) } @@ -700,8 +743,8 @@ func guardBaselineResponse(req *http.Request, response *http.Response) (*http.Re return response, nil } data, err := io.ReadAll(response.Body) - _ = response.Body.Close() - if err != nil || int64(len(data)) > responseBodyLimit { + closeErr := response.Body.Close() + if err != nil || closeErr != nil || int64(len(data)) > responseBodyLimit { c.invalid = true clear(data) return nil, ErrRemote diff --git a/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go new file mode 100644 index 0000000..683702a --- /dev/null +++ b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go @@ -0,0 +1,235 @@ +package livecanary + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" +) + +type baselineCloseErrorBody struct { + io.Reader + err error +} + +func (b baselineCloseErrorBody) Close() error { return b.err } + +func TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails(t *testing.T) { + capture := &baselineWireCapture{ + stage: "runner-observe", + setID: 7, + runnerName: "fixture-runner", + origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", + runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + } + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodGet, "https://api.example/tenant/v2/_apis/distributedtask/pools/0/agents?agentName=fixture-runner&api-version=6.0-preview", nil) + if err != nil { + t.Fatal(err) + } + response, err := guardBaselineResponse(req, &http.Response{ + StatusCode: http.StatusOK, + Body: &baselineCloseErrorBody{ + Reader: strings.NewReader(`{"count":1,"value":[{"id":19,"name":"fixture-runner","runnerScaleSetId":7}]}`), + err: io.ErrClosedPipe, + }, + }) + if !errors.Is(err, ErrRemote) || response != nil { + t.Fatalf("complete runner body with close error = response %v err %v, want remote rejection", response, err) + } + runner, status := capture.runnerFacts() + if runner != nil || status != http.StatusOK || capture.observed() { + t.Fatalf("runner close error published evidence: runner=%+v status=%d observed=%v", runner, status, capture.observed()) + } +} + +type replacingContextRoundTripper struct{ inner http.RoundTripper } + +func (t replacingContextRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return t.inner.RoundTrip(req.Clone(context.Background())) +} + +func TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner(t *testing.T) { + tests := []struct { + name string + capture *baselineWireCapture + method string + target string + body string + status int + response string + }{ + { + name: "set-observe", + capture: &baselineWireCapture{ + stage: "set-observe", setID: 7, organization: "fixture-org", + allowedHosts: []string{"api.example"}, + }, + method: http.MethodGet, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", + status: http.StatusOK, + response: `{}`, + }, + { + name: "runner-observe", + capture: &baselineWireCapture{ + stage: "runner-observe", setID: 7, runnerName: "fixture-runner", + origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + }, + method: http.MethodGet, + target: "https://api.example/tenant/v2/_apis/distributedtask/pools/0/agents?agentName=fixture-runner&api-version=6.0-preview", + status: http.StatusOK, + response: `{}`, + }, + { + name: "session-open", + capture: &baselineWireCapture{ + stage: "session-open", setID: 7, organization: "fixture-org", owner: "fixture-owner", + allowedHosts: []string{"api.example"}, + }, + method: http.MethodPost, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions?api-version=6.0-preview", + body: `{"sessionId":"00000000-0000-0000-0000-000000000000","ownerName":"fixture-owner"}`, + status: http.StatusOK, + response: `{}`, + }, + { + name: "ack", + capture: &baselineWireCapture{ + stage: "ack", setID: 7, sessionID: "session", queue: "https://api.example/tenant/v2/queue?proof=fixture", cursor: 41, + origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + }, + method: http.MethodDelete, + target: "https://api.example/tenant/v2/queue/41?proof=fixture", + status: http.StatusNoContent, + }, + { + name: "acquire", + capture: &baselineWireCapture{ + stage: "acquire", setID: 7, requestIDs: []int64{41}, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + }, + method: http.MethodPost, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", + body: `[41]`, + status: http.StatusOK, + response: `{"count":1,"value":[41]}`, + }, + { + name: "jit", + capture: &baselineWireCapture{ + stage: "jit", setID: 7, origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + }, + method: http.MethodPost, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/generatejitconfig?api-version=6.0-preview", + status: http.StatusOK, + response: `{"encodedJITConfig":"AAAAAAAAAAAAAAAAAAAAAA=="}`, + }, + { + name: "terminal-session-close", + capture: &baselineWireCapture{ + stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + }, + method: http.MethodDelete, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", + status: http.StatusNoContent, + }, + { + name: "terminal-set", + capture: &baselineWireCapture{ + stage: "terminal-set", setID: 7, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + }, + method: http.MethodGet, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", + status: http.StatusOK, + response: `{}`, + }, + { + name: "terminal-set-recheck", + capture: &baselineWireCapture{ + stage: "terminal-set-recheck", setID: 7, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + }, + method: http.MethodGet, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", + status: http.StatusOK, + response: `{}`, + }, + { + name: "terminal-set-absence", + capture: &baselineWireCapture{ + stage: "terminal-set-absence", setID: 7, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + }, + method: http.MethodGet, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", + status: http.StatusNotFound, + }, + { + name: "terminal-set-delete", + capture: &baselineWireCapture{ + stage: "terminal-set-delete", setID: 7, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + }, + method: http.MethodDelete, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", + status: http.StatusNoContent, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + innerCalls := 0 + inner := drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: tc.status, Body: io.NopCloser(strings.NewReader(tc.response)), Request: req}, nil + }) + transport := responseBudgetTransport{inner: replacingContextRoundTripper{inner: baselineRequestCaptureTransport{inner: inner}}} + req, err := http.NewRequestWithContext(tc.capture.context(context.Background()), tc.method, tc.target, strings.NewReader(tc.body)) + if err != nil { + t.Fatal(err) + } + response, err := transport.RoundTrip(req) + if !errors.Is(err, ErrRemote) || response != nil { + t.Fatalf("replacement context boundary = response %v err %v, want remote rejection", response, err) + } + if innerCalls != 0 { + t.Fatalf("replacement context reached inner transport: calls=%d", innerCalls) + } + }) + } +} + +func TestBaselineWireMarkerIsRemovedBeforeInner(t *testing.T) { + capture := &baselineWireCapture{ + stage: "acquire", setID: 7, requestIDs: []int64{41}, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + inner := drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + if values := baselineWireMarkerValues(req); len(values) != 0 { + t.Fatalf("private marker reached inner transport: %v", values) + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"count":1,"value":[41]}`)), Request: req}, nil + }) + transport := responseBudgetTransport{inner: baselineRequestCaptureTransport{inner: inner}} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", strings.NewReader("[41]")) + if err != nil { + t.Fatal(err) + } + if _, err := transport.RoundTrip(req); err != nil { + t.Fatalf("valid marked acquisition = %v", err) + } + if innerCalls != 1 || !capture.observed() { + t.Fatalf("valid marked acquisition calls=%d observed=%v, want one accepted request", innerCalls, capture.observed()) + } +} diff --git a/experiments/g01-scaleset/livecanary/response_budget.go b/experiments/g01-scaleset/livecanary/response_budget.go index 7966a20..7244014 100644 --- a/experiments/g01-scaleset/livecanary/response_budget.go +++ b/experiments/g01-scaleset/livecanary/response_budget.go @@ -36,22 +36,31 @@ func withResponseBudget(transport *http.Transport, wrappers ...func(http.RoundTr type responseBudgetTransport struct{ inner http.RoundTripper } func (t responseBudgetTransport) RoundTrip(req *http.Request) (*http.Response, error) { + wireReq := req + if req != nil { + if c, _ := req.Context().Value(baselineWireKey{}).(*baselineWireCapture); c != nil { + wireReq = markBaselineWireRequest(req) + } + } // No phase intentionally PATCHes. In particular, stop SDK 401 refresh BEFORE // it can replace the session/queue and retry an ACK under a different owner. - if req.Method == http.MethodPatch { + if wireReq == nil { + return nil, ErrRemote + } + if wireReq.Method == http.MethodPatch { return nil, ErrQuarantine } - response, err := t.inner.RoundTrip(req) + response, err := t.inner.RoundTrip(wireReq) if err != nil { return nil, err } - captureRunnerResponse(req, response.StatusCode) + captureRunnerResponse(wireReq, response.StatusCode) if response.ContentLength > responseBodyLimit { _ = response.Body.Close() return nil, errResponseBudget } response.Body = &responseBudgetBody{source: response.Body, remaining: responseBodyLimit} - return guardBaselineResponse(req, response) + return guardBaselineResponse(wireReq, response) } // Read at most the budget plus one detection byte, including decoded gzip and From c2ebdafecbf5f8e01040a89279e7bc3766ba1a7e Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 08:26:01 +0900 Subject: [PATCH 53/63] docs(g01): record wire boundary P1 evidence --- docs/evidence/g01-idle-drain.md | 94 +++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 3e7a29d..9339238 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2230,3 +2230,97 @@ commit. Rollback is recoverable with documentation append separately if needed. No live App/runner/canary evidence is claimed; the unresolved live G01 gate, independent exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: runner close failure and wrapper marker propagation + +Date: 2026-09-13. This correction started from exact head +`a8bfc277e7b663034dc1903bd9e8d758e3b0c88b` and addresses the fresh Codex P1 +[runner response close failure finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997904130) +and [marked request context replacement finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997904133). +No Project, Issue, goal, dependency, status or branch field was changed; no +review request, live App/runner/canary operation, workflow replay, Docker/Lima, +Keychain, launchd, credential or cleanup operation was performed. + +#### Red-first reproduction + +The meaningful regressions were added before the source correction and run +against the unchanged starting head. This focused command exited 1: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails|TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner)$' -count=1 -v +``` + +The runner case failed because complete JSON followed by `Close` returning +`io.ErrClosedPipe` still returned a response with nil error and published +runner facts. The initial replacement-context matrix failed because the set, +runner, session-open, ACK, acquisition, JIT and terminal-session-close marked +requests either returned a response or reached the inner transport; the runner +case reported one inner call. The final regression matrix also covers the +terminal-set variants. The assertions retain only bounded status/error/ +observation state and inner-call counts; no response body, Authorization +value, token, URL or private log is retained. + +#### Minimal correction and green evidence + +`guardBaselineResponse` now checks the body `Close` result alongside the read +result and size before decoding or publishing any runner facts. Clean EOF is +still accepted, while non-EOF read errors remain fail-closed. The shared +response-budget boundary clones only marked requests and adds an ephemeral +private transport marker; `Request.Clone`-style wrappers preserve that marker, +and the final baseline transport rejects a marker whose private operation +context was lost before calling its inner transport. The marker is deleted at +that final boundary and carries no operation, credential, response or token +data, so valid pinned-SDK traffic and intentionally unmarked forwarding remain +unchanged. + +The focused normal command exited 0 in 0.626s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails|TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner|TestBaselineWireMarkerIsRemovedBeforeInner|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=300s +``` + +The corresponding focused race command exited 0 in 1.755s with no race +diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails|TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner|TestBaselineWireMarkerIsRemovedBeforeInner|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -timeout=300s +``` + +The full `livecanary` package normal run exited 0 in 26.544s and the full race +run exited 0 in 38.923s, with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=360s +``` + +`GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary`, +`GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check`, and `git diff --check` +each exited 0. The diff secret/private-path scan exited 0 and printed +`diff secret/private-path scan passed`: + +```text +set -e +if git show --format= --text 3c8567795c69ec3d4b0171b685d0ac21506f83f5 -- experiments/g01-scaleset/livecanary/baseline_wire.go experiments/g01-scaleset/livecanary/response_budget.go experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go | rg -n -i '(/Users/|/home/|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})'; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The exact implementation source head is +`3c8567795c69ec3d4b0171b685d0ac21506f83f5` (`fix(g01): fence marked wire +requests across wrappers`). The exact two-module offline gate was run against +that source head and exited 0 with `offline experiment checks passed: 2 +module(s)`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +Rollback is recoverable with `git revert --no-edit +3c8567795c69ec3d4b0171b685d0ac21506f83f5`; this evidence append is a separate +documentation-only commit and can be reverted independently. No live +App/runner/canary evidence is claimed; the unresolved live G01 gate, +independent exact-head Codex review, CI and merge remain coordinator-owned. From 6b025bc1d07f67b2acb4c6ad9ad14429f195490a Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 08:59:13 +0900 Subject: [PATCH 54/63] fix(g01): reject evidence delete close failures --- .../g01-scaleset/livecanary/baseline_wire.go | 6 ++ .../livecanary/baseline_wire_p1_test.go | 79 +++++++++++++++++++ 2 files changed, 85 insertions(+) diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index d652cb8..e150cf9 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -740,6 +740,12 @@ func guardBaselineResponse(req *http.Request, response *http.Response) (*http.Re return nil, ErrRemote } if response.StatusCode != http.StatusOK { + if response.StatusCode == http.StatusNoContent && (c.stage == "ack" || c.stage == "terminal-session-close" || c.stage == "terminal-set-delete") { + if closeErr := response.Body.Close(); closeErr != nil { + c.invalid = true + return nil, ErrRemote + } + } return response, nil } data, err := io.ReadAll(response.Body) diff --git a/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go index 683702a..4195c18 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go @@ -46,6 +46,85 @@ func TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails(t *testing.T) } } +func TestGuardBaselineResponseRejectsTerminalCloseWhenBodyCloseFails(t *testing.T) { + capture := &baselineWireCapture{ + stage: "terminal-session-close", + setID: 7, + sessionID: "session", + origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", + runtimePathPrefixSet: true, + } + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodDelete, "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", nil) + if err != nil { + t.Fatal(err) + } + response, err := guardBaselineResponse(req, &http.Response{ + StatusCode: http.StatusNoContent, + Body: &baselineCloseErrorBody{Reader: strings.NewReader(""), err: io.ErrClosedPipe}, + }) + if !errors.Is(err, ErrRemote) || response != nil { + t.Fatalf("terminal close body with close error = response %v err %v, want remote rejection", response, err) + } + _, _, _, status := capture.facts() + if status != http.StatusNoContent || capture.observed() { + t.Fatalf("terminal close body close error published evidence: status=%d observed=%v", status, capture.observed()) + } +} + +func TestGuardBaselineResponseRejectsEvidenceDeleteWhenBodyCloseFails(t *testing.T) { + tests := []struct { + name string + capture *baselineWireCapture + target string + }{ + { + name: "ack", + capture: &baselineWireCapture{ + stage: "ack", setID: 7, sessionID: "session", queue: "https://api.example/tenant/v2/queue?proof=fixture", cursor: 41, + origin: "https://api.example:443", runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + }, + target: "https://api.example/tenant/v2/queue/41?proof=fixture", + }, + { + name: "terminal-session-close", + capture: &baselineWireCapture{ + stage: "terminal-session-close", setID: 7, sessionID: "session", origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, + }, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/sessions/session?api-version=6.0-preview", + }, + { + name: "terminal-set-delete", + capture: &baselineWireCapture{ + stage: "terminal-set-delete", setID: 7, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + }, + target: "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequestWithContext(tc.capture.context(context.Background()), http.MethodDelete, tc.target, nil) + if err != nil { + t.Fatal(err) + } + response, err := guardBaselineResponse(req, &http.Response{ + StatusCode: http.StatusNoContent, + Body: &baselineCloseErrorBody{Reader: strings.NewReader(""), err: io.ErrClosedPipe}, + }) + if !errors.Is(err, ErrRemote) || response != nil { + t.Fatalf("evidence DELETE body with close error = response %v err %v, want remote rejection", response, err) + } + _, _, _, status := tc.capture.facts() + if status != http.StatusNoContent || tc.capture.observed() { + t.Fatalf("evidence DELETE close error published evidence: status=%d observed=%v", status, tc.capture.observed()) + } + }) + } +} + type replacingContextRoundTripper struct{ inner http.RoundTripper } func (t replacingContextRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { From 6e727bf5c0d276227878b5808ab0211ddb9f3e15 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 09:01:27 +0900 Subject: [PATCH 55/63] docs(g01): record expected-204 close failure evidence --- docs/evidence/g01-idle-drain.md | 85 +++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 9339238..3afe296 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2324,3 +2324,88 @@ Rollback is recoverable with `git revert --no-edit documentation-only commit and can be reverted independently. No live App/runner/canary evidence is claimed; the unresolved live G01 gate, independent exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: expected-204 evidence DELETE close failure + +Date: 2026-09-13. This correction started from exact head +`c2ebdafecbf5f8e01040a89279e7bc3766ba1a7e` and addresses the fresh Codex P1 +[successful expected-204 ACK/session-close response body close failure finding](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998020455). +No Project, Issue, goal, dependency, status or branch field was changed; no +review request, live App/runner/canary operation, workflow replay, Docker/Lima, +Keychain, launchd, credential or cleanup operation was performed. + +#### Red-first reproduction + +The terminal session-close regression was added before the source correction +and run against the unchanged starting implementation. This focused command +exited 1 as expected: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestGuardBaselineResponseRejectsTerminalCloseWhenBodyCloseFails$' -count=1 -v +``` + +The test reported a non-nil 204 response with `err ` instead of +`ErrRemote`; the capture consequently remained observed. The regression only +retains bounded status/error/observation state and the synthetic +`io.ErrClosedPipe`; it records no response body, Authorization value, token, +URL or private log. + +#### Minimal correction and green evidence + +`guardBaselineResponse` now validates `response.Body.Close()` for successful +204 evidence DELETE stages (`ack`, `terminal-session-close` and +`terminal-set-delete`) before allowing the response to remain usable. A close +failure marks the capture invalid and returns `ErrRemote`, so `wire.observed` +cannot publish evidence; clean 204 bodies remain accepted. The 202 poll path, +unmarked forwarding and existing live-operation restrictions remain +unchanged. The boundary matrix covers ACK, terminal session close and terminal +scale-set deletion, alongside the existing runner close-error regression. + +The focused normal command exited 0 in 0.478s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails|TestGuardBaselineResponseRejectsTerminalCloseWhenBodyCloseFails|TestGuardBaselineResponseRejectsEvidenceDeleteWhenBodyCloseFails|TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner|TestBaselineWireMarkerIsRemovedBeforeInner|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -v -timeout=300s +``` + +The corresponding focused race command exited 0 in 1.794s with no race +diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails|TestGuardBaselineResponseRejectsTerminalCloseWhenBodyCloseFails|TestGuardBaselineResponseRejectsEvidenceDeleteWhenBodyCloseFails|TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner|TestBaselineWireMarkerIsRemovedBeforeInner|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -timeout=300s +``` + +The full `livecanary` package normal run exited 0 in 26.951s and the full race +run exited 0 in 38.174s, with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=360s +``` + +The offline/static checks all exited 0: the two-module offline gate printed +`offline experiment checks passed: 2 module(s)`, `go vet ./livecanary`, +`gofmt.sh check`, `git diff --check`, and the diff secret/private-path scan +printed `diff secret/private-path scan passed`. + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +(cd experiments/g01-scaleset && GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary) +GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check +git diff --check +set -e +if git diff --text 6b025bc1d07f67b2acb4c6ad9ad14429f195490a^ 6b025bc1d07f67b2acb4c6ad9ad14429f195490a -- experiments/g01-scaleset/livecanary/baseline_wire.go experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go | rg -n -i '(/Users/|/home/|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})'; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The exact implementation source head is +`6b025bc1d07f67b2acb4c6ad9ad14429f195490a` (`fix(g01): reject evidence delete +close failures`); this evidence append is a separate documentation-only +commit. Rollback is recoverable with `git revert --no-edit +6b025bc1d07f67b2acb4c6ad9ad14429f195490a`; revert this documentation append +separately if needed. No live App/runner/canary evidence is claimed; the +unresolved live G01 gate, independent exact-head Codex review, CI and merge +remain coordinator-owned. From d2a2be896844d95c61dfd955bd027469b315d36e Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 09:37:35 +0900 Subject: [PATCH 56/63] fix(g01): bind JIT wire and terminal absence evidence --- .../livecanary/baseline_execution.go | 12 +- .../g01-scaleset/livecanary/baseline_wire.go | 37 +++- .../livecanary/baseline_wire_p1_test.go | 181 ++++++++++++++++++ 3 files changed, 224 insertions(+), 6 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_execution.go b/experiments/g01-scaleset/livecanary/baseline_execution.go index 293de22..b3c56b1 100644 --- a/experiments/g01-scaleset/livecanary/baseline_execution.go +++ b/experiments/g01-scaleset/livecanary/baseline_execution.go @@ -20,7 +20,17 @@ func (s *pairedBaselineScope) afterAcquire(ctx context.Context, acquisition base if err != nil { return err } - capture := &baselineWireCapture{stage: "jit", setID: s.setID} + if s.listener == nil { + return ErrQuarantine + } + // AcquireJobs invokes this continuation while the listener mutex is held. + // wire reads the already-captured session tuple directly for this handoff; + // taking the listener accessors here would recursively lock that mutex. + capture := s.listener.wire("jit") + capture.runnerName = s.approval.workerName() + if capture.origin == "" || !capture.runtimePathPrefixSet || capture.runtimePathPrefix == "" || capture.runnerName == "" { + return ErrQuarantine + } request, cancel := context.WithTimeout(ctx, operationTimeout) got, callErr := s.captured.GenerateJIT(capture.context(request), s.setID, s.approval.workerName()) cancel() diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index e150cf9..61dcc5b 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -165,7 +165,6 @@ func (c *baselineWireCapture) target(r *http.Request) bool { u.Path += "/" + strconv.Itoa(c.cursor) return r.Method == http.MethodDelete && r.URL.String() == u.String() } - suffix := "/runnerscalesets/" + strconv.Itoa(c.setID) + "/" if c.stage == "set-observe" || c.stage == "terminal-set" || c.stage == "terminal-set-recheck" || c.stage == "terminal-set-absence" || c.stage == "terminal-set-delete" { if !c.snapshotOriginAllowed(r) { return false @@ -195,11 +194,10 @@ func (c *baselineWireCapture) target(r *http.Request) bool { } return r.Method == "POST" && c.runtimeRequestTarget(r, "acquirejobs") && baselineExactAPIVersionQuery(r.URL.RawQuery) } else if c.stage == "jit" { - suffix += "generatejitconfig" + return c.origin != "" && c.runtimePathPrefixSet && c.runtimePathPrefix != "" && c.snapshotOriginAllowed(r) && r.Method == http.MethodPost && c.runtimeRequestTarget(r, "generatejitconfig") && baselineExactAPIVersionQuery(r.URL.RawQuery) } else { return false } - return r.Method == "POST" && strings.HasSuffix(r.URL.Path, suffix) && baselineExactAPIVersionQuery(r.URL.RawQuery) } // markedDeleteIdentityReady requires the private identity captured by the @@ -544,6 +542,22 @@ func (c *baselineWireCapture) captureRequest(req *http.Request) error { c.mu.Unlock() return nil } + if c.stage == "jit" { + if !c.target(req) { + return c.rejectRequest(req) + } + if err := c.reserveOneShot(req); err != nil { + return err + } + data, err := readBaselineRequestBody(req.Body) + req.Body = nil + if err != nil || !validBaselineJITBody(data, c.runnerName) { + clear(data) + return c.rejectReservedRequest(req) + } + installBaselineRequestBody(req, data) + return nil + } if c.stage == "set-observe" || c.stage == "runner-observe" || strings.HasPrefix(c.stage, "terminal-set") { if !snapshotRequestCandidate(req) { if c.sessionOpenBootstrapTarget(req) { @@ -636,6 +650,17 @@ func validBaselineSessionOpenBody(data []byte, owner string) bool { return DecodeStrict(data, &body) == nil && body.SessionID != nil && *body.SessionID == "00000000-0000-0000-0000-000000000000" && body.Owner != nil && *body.Owner == owner } +func validBaselineJITBody(data []byte, runnerName string) bool { + if runnerName == "" { + return false + } + var body struct { + Name *string `json:"name"` + WorkFolder *string `json:"workFolder"` + } + return DecodeStrict(data, &body) == nil && body.Name != nil && *body.Name == runnerName && body.WorkFolder != nil && *body.WorkFolder == "_work" +} + func installBaselineRequestBody(req *http.Request, data []byte) { if req == nil { clear(data) @@ -740,8 +765,10 @@ func guardBaselineResponse(req *http.Request, response *http.Response) (*http.Re return nil, ErrRemote } if response.StatusCode != http.StatusOK { - if response.StatusCode == http.StatusNoContent && (c.stage == "ack" || c.stage == "terminal-session-close" || c.stage == "terminal-set-delete") { - if closeErr := response.Body.Close(); closeErr != nil { + expectedEmptyBody := response.StatusCode == http.StatusNoContent && (c.stage == "ack" || c.stage == "terminal-session-close" || c.stage == "terminal-set-delete") + expectedAbsentBody := response.StatusCode == http.StatusNotFound && c.stage == "terminal-set-absence" + if expectedEmptyBody || expectedAbsentBody { + if response.Body == nil || response.Body.Close() != nil { c.invalid = true return nil, ErrRemote } diff --git a/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go index 4195c18..8b45839 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go @@ -125,6 +125,187 @@ func TestGuardBaselineResponseRejectsEvidenceDeleteWhenBodyCloseFails(t *testing } } +func TestGuardBaselineResponseRejectsTerminalSetAbsenceWhenBodyCloseFails(t *testing.T) { + capture := &baselineWireCapture{ + stage: "terminal-set-absence", + setID: 7, + origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", + runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + } + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodGet, "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", nil) + if err != nil { + t.Fatal(err) + } + response, err := guardBaselineResponse(req, &http.Response{ + StatusCode: http.StatusNotFound, + Body: &baselineCloseErrorBody{Reader: strings.NewReader(""), err: io.ErrClosedPipe}, + }) + if !errors.Is(err, ErrRemote) || response != nil { + t.Fatalf("terminal set absence body with close error = response %v err %v, want remote rejection", response, err) + } + _, _, _, status := capture.facts() + if status != http.StatusNotFound || capture.observed() { + t.Fatalf("terminal set absence close error published evidence: status=%d observed=%v", status, capture.observed()) + } +} + +func TestGuardBaselineResponsePreservesTerminalSetAbsenceWhenBodyCloseSucceeds(t *testing.T) { + capture := &baselineWireCapture{ + stage: "terminal-set-absence", + setID: 7, + origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", + runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + } + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodGet, "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7?api-version=6.0-preview", nil) + if err != nil { + t.Fatal(err) + } + response, err := guardBaselineResponse(req, &http.Response{ + StatusCode: http.StatusNotFound, + Body: &baselineCloseErrorBody{Reader: strings.NewReader(""), err: nil}, + }) + if err != nil || response == nil || response.StatusCode != http.StatusNotFound || !capture.observed() { + t.Fatalf("valid terminal set absence = response %v err %v observed=%v, want retained 404 evidence", response, err, capture.observed()) + } +} + +type baselineRequestMutationRoundTripper struct { + inner http.RoundTripper + mutate func(*http.Request) +} + +func (t baselineRequestMutationRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + if t.mutate != nil { + t.mutate(req) + } + return t.inner.RoundTrip(req) +} + +type baselineJITRoundTripper func(*http.Request) (*http.Response, error) + +func (f baselineJITRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + +func TestBaselineMarkedJITRequestRejectsPhysicalTupleMutationBeforeInner(t *testing.T) { + const ( + origin = "https://api.example:443" + prefix = "/tenant/v2" + runnerName = "g01-test-worker-1" + target = "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/generatejitconfig?api-version=6.0-preview" + requestBody = `{"name":"g01-test-worker-1","workFolder":"_work"}` + foreignBody = `{"name":"foreign-worker","workFolder":"_work"}` + ) + for _, tc := range []struct { + name string + mutate func(*http.Request) + }{ + { + name: "wrong physical origin", + mutate: func(req *http.Request) { + req.URL.Host = "other.example" + req.Host = req.URL.Host + req.URL.RawPath = "" + }, + }, + { + name: "wrong runtime tenant prefix", + mutate: func(req *http.Request) { + req.URL.Path = "/tenant/foreign/_apis/runtime/runnerscalesets/7/generatejitconfig" + req.URL.RawPath = "" + }, + }, + { + name: "wrong request body", + mutate: func(req *http.Request) { + req.Body = io.NopCloser(strings.NewReader(foreignBody)) + req.ContentLength = int64(len(foreignBody)) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "jit", + setID: 7, + runnerName: runnerName, + origin: origin, + runtimePathPrefix: prefix, + runtimePathPrefixSet: true, + allowedHosts: []string{"api.example", "other.example"}, + } + innerCalls := 0 + inner := baselineJITRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"encodedJITConfig":"AAAAAAAAAAAAAAAAAAAAAA=="}`)), Request: req}, nil + }) + transport := responseBudgetTransport{inner: baselineRequestMutationRoundTripper{ + inner: baselineRequestCaptureTransport{inner: inner}, + mutate: tc.mutate, + }} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, target, strings.NewReader(requestBody)) + if err != nil { + t.Fatal(err) + } + response, err := transport.RoundTrip(req) + if !errors.Is(err, ErrRemote) || response != nil { + t.Fatalf("mutated marked JIT = response %v err %v, want remote rejection", response, err) + } + if innerCalls != 0 { + t.Fatalf("mutated marked JIT reached inner transport: calls=%d", innerCalls) + } + if capture.requestObserved() || capture.observed() { + t.Fatalf("mutated marked JIT published evidence: requestObserved=%v observed=%v", capture.requestObserved(), capture.observed()) + } + }) + } +} + +func TestBaselineJITPreservesValidMarkedAndUnmarkedForwarding(t *testing.T) { + const ( + origin = "https://api.example:443" + prefix = "/tenant/v2" + runnerName = "g01-test-worker-1" + target = "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/generatejitconfig?api-version=6.0-preview" + requestBody = `{"name":"g01-test-worker-1","workFolder":"_work"}` + ) + capture := &baselineWireCapture{ + stage: "jit", + setID: 7, + runnerName: runnerName, + origin: origin, + runtimePathPrefix: prefix, + runtimePathPrefixSet: true, + allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + inner := baselineJITRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"encodedJITConfig":"AAAAAAAAAAAAAAAAAAAAAA=="}`)), Request: req}, nil + }) + transport := responseBudgetTransport{inner: baselineRequestCaptureTransport{inner: inner}} + marked, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, target, strings.NewReader(requestBody)) + if err != nil { + t.Fatal(err) + } + response, err := transport.RoundTrip(marked) + if err != nil || response == nil || response.StatusCode != http.StatusOK || innerCalls != 1 || !capture.observed() { + t.Fatalf("valid marked JIT = response %v err %v inner calls=%d observed=%v, want one accepted request", response, err, innerCalls, capture.observed()) + } + + unmarked, err := http.NewRequest(http.MethodPost, "https://other.example/rewritten", strings.NewReader(`{"name":"foreign-worker","workFolder":"foreign"}`)) + if err != nil { + t.Fatal(err) + } + response, err = transport.RoundTrip(unmarked) + if err != nil || response == nil || response.StatusCode != http.StatusOK || innerCalls != 2 { + t.Fatalf("unmarked forwarding = response %v err %v inner calls=%d, want forwarding", response, err, innerCalls) + } +} + type replacingContextRoundTripper struct{ inner http.RoundTripper } func (t replacingContextRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { From fe666a9dcfcab15ac3a95ebfe588374ba7695246 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 09:49:48 +0900 Subject: [PATCH 57/63] docs(g01): record terminal and JIT wire findings --- docs/evidence/g01-idle-drain.md | 91 +++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 3afe296..9bb329e 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2409,3 +2409,94 @@ commit. Rollback is recoverable with `git revert --no-edit separately if needed. No live App/runner/canary evidence is claimed; the unresolved live G01 gate, independent exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: terminal absence close failure and marked JIT wire tuple + +Date: 2026-09-13. This correction started from exact head +`6e727bf5c0d276227878b5808ab0211ddb9f3e15` and addresses the two fresh Codex +P1 findings on PR #72: [terminal-set-absence expected-404 response body close failure](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998106046) +and [marked JIT request origin/prefix/body mutation](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998106048). +No Issue, Project, goal, dependency, status or branch field was changed; no +live App/runner/canary operation, workflow replay, Docker/Lima, Keychain, +launchd, credential or cleanup operation was performed. + +#### Red-first reproduction + +Both regressions were added and run before the source correction against the +unchanged exact starting head. This focused command exited 1 as expected: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestGuardBaselineResponseRejectsTerminalSetAbsenceWhenBodyCloseFails|TestBaselineMarkedJITRequestRejectsPhysicalTupleMutationBeforeInner)$' -count=1 -v +``` + +The terminal absence test observed a non-nil 404 response with `err ` and +the capture still observed. The three JIT mutation cases (physical origin, +tenant prefix and request body) each returned a successful response and +reached the inner transport instead of rejecting before forwarding. The red +tests retain only bounded status, error and observation state plus the +synthetic `io.ErrClosedPipe`; no response body, Authorization value, token, +URL or private log is recorded. + +#### Minimal correction and green evidence + +`guardBaselineResponse` now close-checks expected `terminal-set-absence` 404 +bodies before accepting absence evidence; a close failure marks the capture +invalid and returns `ErrRemote`, while a clean 404 remains accepted. Marked +JIT requests now require the already captured session origin, exact runtime +tenant prefix, scale-set route and API-version query, and the bounded strict +JSON body with the approved runner name and `_work` folder. The JIT capture is +seeded from the listener-held session tuple without recursively taking its +mutex; valid marked JIT and unmarked forwarding remain covered. + +The focused normal command exited 0 in 0.630s, and the focused race command +exited 0 in 1.710s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails|TestGuardBaselineResponseRejectsTerminalCloseWhenBodyCloseFails|TestGuardBaselineResponseRejectsTerminalSetAbsenceWhenBodyCloseFails|TestGuardBaselineResponsePreservesTerminalSetAbsenceWhenBodyCloseSucceeds|TestGuardBaselineResponseRejectsEvidenceDeleteWhenBodyCloseFails|TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner|TestBaselineWireMarkerIsRemovedBeforeInner|TestBaselineMarkedJITRequestRejectsPhysicalTupleMutationBeforeInner|TestBaselineJITPreservesValidMarkedAndUnmarkedForwarding|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails|TestGuardBaselineResponseRejectsTerminalCloseWhenBodyCloseFails|TestGuardBaselineResponseRejectsTerminalSetAbsenceWhenBodyCloseFails|TestGuardBaselineResponsePreservesTerminalSetAbsenceWhenBodyCloseSucceeds|TestGuardBaselineResponseRejectsEvidenceDeleteWhenBodyCloseFails|TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner|TestBaselineWireMarkerIsRemovedBeforeInner|TestBaselineMarkedJITRequestRejectsPhysicalTupleMutationBeforeInner|TestBaselineJITPreservesValidMarkedAndUnmarkedForwarding|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainRejectsAcquireTargetMutationBeforeFixture|TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete|TestDriverDrainThroughPinnedSDKAndPollHook)$' -count=1 -timeout=300s +``` + +The full `livecanary` package normal run exited 0 in 25.467s and the full race +run exited 0 in 36.955s, with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=360s +``` + +The offline paired-terminal integration matrix, which exercises the listener +seeded valid JIT path without live resources, exited 0 in 13.905s normally and +41.479s under race: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture ./livecanary -run '^(TestPairedTerminalActualJournalsFinalize|TestPairedTerminalCompletionCadenceAndReceiptSeparation|TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' -count=1 -v -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture -race ./livecanary -run '^(TestPairedTerminalActualJournalsFinalize|TestPairedTerminalCompletionCadenceAndReceiptSeparation|TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' -count=1 -timeout=360s +``` + +The required two-module offline/static gate exited 0 and printed +`offline experiment checks passed: 2 module(s)`. The changed-boundary vet, +format, diff and secret/private-path checks each exited 0; the final scan +printed `diff secret/private-path scan passed`: + +```text +GOWORK=off GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +(cd experiments/g01-scaleset && GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary) +GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check +git diff --check +set -e +if git diff --text d2a2be896844d95c61dfd955bd027469b315d36e^ d2a2be896844d95c61dfd955bd027469b315d36e -- experiments/g01-scaleset/livecanary/baseline_wire.go experiments/g01-scaleset/livecanary/baseline_execution.go experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go | rg -n -i '(/Users/|/home/|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})'; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The exact implementation source/test head is +`d2a2be896844d95c61dfd955bd027469b315d36e` (`fix(g01): bind JIT wire and +terminal absence evidence`); this evidence append is a separate +documentation-only commit. Source rollback is recoverable with +`git revert --no-edit d2a2be896844d95c61dfd955bd027469b315d36e`; revert this +documentation append separately if needed. No live App/runner/canary evidence +is claimed; the unresolved live G01 gate, independent exact-head Codex review, +CI and merge remain coordinator-owned. From 0aabf2bf65ef4e9c0a5e944ca8b56d76209aae0d Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 10:07:25 +0900 Subject: [PATCH 58/63] fix(g01): reject workflow verification close failures --- .../g01-scaleset/livecanary/observer_http.go | 6 ++-- .../livecanary/sdk_integration_test.go | 28 +++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/observer_http.go b/experiments/g01-scaleset/livecanary/observer_http.go index 88d45d8..446e622 100644 --- a/experiments/g01-scaleset/livecanary/observer_http.go +++ b/experiments/g01-scaleset/livecanary/observer_http.go @@ -53,10 +53,10 @@ func (a *SDKAPI) observationGET(ctx context.Context, path, token, endpoint strin if err != nil { return out, ErrRemote } - defer resp.Body.Close() out.Status = resp.StatusCode - data, err := io.ReadAll(io.LimitReader(resp.Body, responseBodyLimit+1)) - if err != nil || int64(len(data)) > responseBodyLimit || ctx.Err() != nil { + data, readErr := io.ReadAll(io.LimitReader(resp.Body, responseBodyLimit+1)) + closeErr := resp.Body.Close() + if readErr != nil || closeErr != nil || int64(len(data)) > responseBodyLimit || ctx.Err() != nil { return out, ErrRemote } if resp.StatusCode == http.StatusNotFound { diff --git a/experiments/g01-scaleset/livecanary/sdk_integration_test.go b/experiments/g01-scaleset/livecanary/sdk_integration_test.go index 27f10f7..6c00456 100644 --- a/experiments/g01-scaleset/livecanary/sdk_integration_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_integration_test.go @@ -11,6 +11,7 @@ import ( "net/http" "net/http/httptest" "slices" + "strconv" "strings" "sync" "sync/atomic" @@ -688,6 +689,26 @@ func TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects(t *testing.T) { } } +func TestPinnedSDKDrainRejectsVerifyRunCloseErrorBeforeEffects(t *testing.T) { + a := approval() + fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{verifyCloseError: io.ErrClosedPipe}) + hook.releaseResponse() + j := &memoryJournal{} + d := &Driver{Approval: a, Journal: j, API: base} + client := &journaledDrainClient{d: d, inner: session, sessionID: session.Session().SessionID.String(), setID: 7, hook: hook} + + message, err := client.GetMessage(context.Background(), 0, drainInitialCapacity) + if !errors.Is(err, ErrQuarantine) || message != nil { + t.Fatalf("workflow verification close error = message %v err %v, want quarantine", message, err) + } + if fixture.verifyCalls.Load() != 1 { + t.Fatalf("workflow verification calls = %d, want one", fixture.verifyCalls.Load()) + } + if fixture.acks.Load() != 0 || fixture.acquires.Load() != 0 { + t.Fatalf("workflow verification close error reached effects: ack=%d acquire=%d", fixture.acks.Load(), fixture.acquires.Load()) + } +} + func TestPinnedSDKDrainRejectsSubstitutedSessionAuthorizationBeforeEffects(t *testing.T) { a := approval() fixture, base, session, hook := newPinnedDrainSessionOptions(t, a, pinnedDrainJobBody(a), pinnedDrainOptions{ @@ -1142,6 +1163,7 @@ type pinnedDrainOptions struct { acceptAnyACK bool pollReadError error pollCloseError error + verifyCloseError error snapshotBodies []string mutateRequest func(*http.Request) wrongCloseOrigin bool @@ -1340,6 +1362,12 @@ func newPinnedDrainSessionOptions(t *testing.T, a Approval, body string, options return hook }) } + if options.verifyCloseError != nil { + verifyPath := "/api/v3/repos/" + fixtureApproval.Organization + "/" + fixtureApproval.Repository + "/actions/runs/" + strconv.FormatInt(fixtureApproval.WorkflowRunID, 10) + wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { + return sdkResponseBodyFaultRoundTripper{inner: inner, path: verifyPath, closeErr: options.verifyCloseError} + }) + } if options.wrongCloseOrigin { wrappers = append(wrappers, func(inner http.RoundTripper) http.RoundTripper { return sdkRequestMutationRoundTripper{inner: inner, mutate: func(req *http.Request) { From 5b24636c04b036da238c03efe4e27f983e138251 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 10:26:33 +0900 Subject: [PATCH 59/63] docs(g01): record verification close failure evidence --- docs/evidence/g01-idle-drain.md | 89 +++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index 9bb329e..d97f0ae 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2500,3 +2500,92 @@ documentation-only commit. Source rollback is recoverable with documentation append separately if needed. No live App/runner/canary evidence is claimed; the unresolved live G01 gate, independent exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: workflow verification response body close failure + +Date: 2026-09-13. This correction started from exact PR #72 head +`fe666a9dcfcab15ac3a95ebfe588374ba7695246` and addresses the fresh Codex P1 +finding at [issue comment 5649825382](https://github.com/1XP-AI/gh-runnerd/pull/72#issuecomment-5649825382). +The finding identified that a complete matching workflow-run JSON response +could still have a failing `response.Body.Close`; ignoring that error let +`VerifyRun` authorize the drain poll before ACK/acquisition. No Issue, Project, +goal, dependency, status or live-operation state was changed. + +#### Red-first reproduction + +The regression was run against the exact parent implementation before the +source correction. This command exited 1 as expected: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPinnedSDKDrainRejectsVerifyRunCloseErrorBeforeEffects$' -count=1 +``` + +The failure was meaningful: the pinned drain listener received a non-nil +message with `err ` even though the synthetic verification body returned +`io.ErrClosedPipe` from `Close`; the test expected quarantine. ACK and +acquisition counters remained zero because the regression stops at the +verification gate. The test retains only bounded counters/categories and the +synthetic close-error category; no response body, token, URL, raw SDK error or +private path is recorded. + +#### Minimal correction and green evidence + +`observationGET` now explicitly closes the response after reading it and +requires both read and close success before accepting any observation, +including a matching workflow-run response. It continues returning the +existing opaque `ErrRemote` category, so `VerifyRun` cannot authorize a poll +whose source body close failed; valid verification, clean 404 handling and +unmarked forwarding remain covered by the existing tests. The new pinned SDK +regression confirms the failure quarantines before ACK/acquisition. + +The focused normal command exited 0 in 0.483s; the focused race command exited +0 in 1.795s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestPinnedSDKDrainRejectsVerifyRunCloseErrorBeforeEffects|TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects|TestPinnedSDKDrainRequiresCompletePollStatsBeforeVerifyRun|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainMatchesWireBeforeVerifyRun|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestObserveRejectsIndependentBaseFork|TestObserveJobAttemptRequiresDetailCorroboration)$' -count=1 -v -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestPinnedSDKDrainRejectsVerifyRunCloseErrorBeforeEffects|TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects|TestPinnedSDKDrainRequiresCompletePollStatsBeforeVerifyRun|TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects|TestPinnedSDKDrainRejectsNonEOFPollReadError|TestPinnedSDKDrainMatchesWireBeforeVerifyRun|TestDriverDrainThroughPinnedSDKAndPollHook|TestDrainListenerWithdrawsWhilePollResponseIsHeld|TestObserveRejectsIndependentBaseFork|TestObserveJobAttemptRequiresDetailCorroboration)$' -count=1 -timeout=300s +``` + +The full `livecanary` package normal run exited 0 in 27.162s and the full race +run exited 0 in 37.915s, with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=360s +``` + +The relevant offline paired-terminal fixture matrix exited 0 in 14.588s +normally and 42.410s under race: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture ./livecanary -run '^(TestPairedTerminalActualJournalsFinalize|TestPairedTerminalCompletionCadenceAndReceiptSeparation|TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' -count=1 -v -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -tags=g01_pair_fixture -race ./livecanary -run '^(TestPairedTerminalActualJournalsFinalize|TestPairedTerminalCompletionCadenceAndReceiptSeparation|TestPairedTerminalFinalResultCapacity|TestPairedTerminalPendingChildCapacity|TestPairedTerminalEligibilityUsesFreshExactFacts|TestPairedTerminalCapturedAcknowledgementCancellation|TestPairedTerminalMissingAcknowledgementsAndPostchecks)$' -count=1 -timeout=360s +``` + +The repository's reviewed offline gate exited 0 and printed +`offline experiment checks passed: 2 module(s)`. The changed-module vet, +format, diff and secret/private-path scan each exited 0; the scan printed +`diff secret/private-path scan passed`: + +```text +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +(cd experiments/g01-scaleset && GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary) +GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check +git diff --check +set -e +if git diff --text 0aabf2bf65ef4e9c0a5e944ca8b56d76209aae0d^ 0aabf2bf65ef4e9c0a5e944ca8b56d76209aae0d -- experiments/g01-scaleset/livecanary/observer_http.go experiments/g01-scaleset/livecanary/sdk_integration_test.go | rg -n -i '(/Users/|/home/|-----BEGIN (RSA|OPENSSH|EC|PRIVATE)|github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9_]{20,}|Authorization[^\n]{0,20}Bearer[[:space:]]+[A-Za-z0-9._-]{20,})'; then exit 1; fi +printf '%s\n' 'diff secret/private-path scan passed' +``` + +The exact implementation source/test head is +`0aabf2bf65ef4e9c0a5e944ca8b56d76209aae0d` (`fix(g01): reject workflow +verification close failures`); this evidence append is a separate +documentation-only commit. Source rollback is recoverable with +`git revert --no-edit 0aabf2bf65ef4e9c0a5e944ca8b56d76209aae0d`; revert this +documentation append separately if needed. No live App/runner/canary test, +Docker/Lima, Keychain, launchd or workflow replay was run; independent +exact-head Codex review, CI and merge remain coordinator-owned. From 0f36583535a7f26644e6b58815c8dd0b85b45b8b Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 10:51:34 +0900 Subject: [PATCH 60/63] fix(g01): bind folded baseline wire markers --- .../g01-scaleset/livecanary/baseline_wire.go | 33 ++++-- .../livecanary/baseline_wire_p1_test.go | 106 +++++++++++++++++- 2 files changed, 130 insertions(+), 9 deletions(-) diff --git a/experiments/g01-scaleset/livecanary/baseline_wire.go b/experiments/g01-scaleset/livecanary/baseline_wire.go index 61dcc5b..3d7fd88 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire.go @@ -384,11 +384,28 @@ func (c *baselineWireCapture) authorizationMatches(req *http.Request) bool { const baselineWireMarkerHeader = "X-GH-Runnerd-Baseline-Wire" const baselineWireMarkerValue = "1" -func baselineWireMarkerValues(req *http.Request) []string { +func baselineWireMarkerValues(req *http.Request) (values []string, keyCount int) { if req == nil || req.Header == nil { - return nil + return nil, 0 + } + for key, keyValues := range req.Header { + if strings.EqualFold(key, baselineWireMarkerHeader) { + keyCount++ + values = append(values, keyValues...) + } + } + return values, keyCount +} + +func removeBaselineWireMarker(req *http.Request) { + if req == nil || req.Header == nil { + return + } + for key := range req.Header { + if strings.EqualFold(key, baselineWireMarkerHeader) { + delete(req.Header, key) + } } - return req.Header.Values(baselineWireMarkerHeader) } func markBaselineWireRequest(req *http.Request) *http.Request { @@ -421,20 +438,20 @@ func (t baselineRequestCaptureTransport) RoundTrip(req *http.Request) (*http.Res return nil, ErrRemote } c, _ := req.Context().Value(baselineWireKey{}).(*baselineWireCapture) - markerValues := baselineWireMarkerValues(req) - if c == nil && len(markerValues) > 0 { + markerValues, markerKeyCount := baselineWireMarkerValues(req) + if c == nil && markerKeyCount != 0 { if req.Body != nil { _ = req.Body.Close() } return nil, ErrRemote } if c != nil { - if len(markerValues) > 0 { - if len(markerValues) != 1 || markerValues[0] != baselineWireMarkerValue { + if markerKeyCount != 0 { + if markerKeyCount != 1 || len(markerValues) != 1 || markerValues[0] != baselineWireMarkerValue { return nil, c.rejectRequest(req) } // Never expose the private marker to the physical transport. - req.Header.Del(baselineWireMarkerHeader) + removeBaselineWireMarker(req) } if !baselineRequestHostMatchesURL(req) { return nil, c.rejectRequest(req) diff --git a/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go index 8b45839..e8928c9 100644 --- a/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go +++ b/experiments/g01-scaleset/livecanary/baseline_wire_p1_test.go @@ -312,6 +312,12 @@ func (t replacingContextRoundTripper) RoundTrip(req *http.Request) (*http.Respon return t.inner.RoundTrip(req.Clone(context.Background())) } +type preservingContextRoundTripper struct{ inner http.RoundTripper } + +func (t preservingContextRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return t.inner.RoundTrip(req.Clone(req.Context())) +} + func TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner(t *testing.T) { tests := []struct { name string @@ -476,7 +482,7 @@ func TestBaselineWireMarkerIsRemovedBeforeInner(t *testing.T) { innerCalls := 0 inner := drainRoundTripper(func(req *http.Request) (*http.Response, error) { innerCalls++ - if values := baselineWireMarkerValues(req); len(values) != 0 { + if values, keyCount := baselineWireMarkerValues(req); keyCount != 0 || len(values) != 0 { t.Fatalf("private marker reached inner transport: %v", values) } return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"count":1,"value":[41]}`)), Request: req}, nil @@ -493,3 +499,101 @@ func TestBaselineWireMarkerIsRemovedBeforeInner(t *testing.T) { t.Fatalf("valid marked acquisition calls=%d observed=%v, want one accepted request", innerCalls, capture.observed()) } } + +func rekeyBaselineWireMarker(req *http.Request) { + if req == nil || req.Header == nil { + return + } + var values []string + found := false + for key, keyValues := range req.Header { + if !strings.EqualFold(key, baselineWireMarkerHeader) { + continue + } + found = true + values = append(values, keyValues...) + delete(req.Header, key) + } + if !found { + return + } + rekeyed := strings.ToLower(baselineWireMarkerHeader) + req.Header[rekeyed] = values +} + +func TestBaselineMarkedCaseFoldedMarkerWithReplacementContextRejectsBeforeInner(t *testing.T) { + capture := &baselineWireCapture{ + stage: "acquire", setID: 7, requestIDs: []int64{41}, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + inner := drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + return &http.Response{StatusCode: http.StatusOK, Body: http.NoBody, Request: req}, nil + }) + transport := responseBudgetTransport{inner: baselineRequestMutationRoundTripper{ + inner: replacingContextRoundTripper{inner: baselineRequestCaptureTransport{inner: inner}}, + mutate: rekeyBaselineWireMarker, + }} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://other.example/foreign", strings.NewReader("[99]")) + if err != nil { + t.Fatal(err) + } + response, err := transport.RoundTrip(req) + if !errors.Is(err, ErrRemote) || response != nil { + t.Fatalf("case-folded marker with replacement context = response %v err %v, want remote rejection", response, err) + } + if innerCalls != 0 { + t.Fatalf("case-folded marker with replacement context reached inner transport: calls=%d", innerCalls) + } + if capture.requestObserved() || capture.observed() { + t.Fatalf("case-folded marker with replacement context published evidence: requestObserved=%v observed=%v", capture.requestObserved(), capture.observed()) + } +} + +func TestBaselineMarkedCaseFoldedMarkerPreservesContextAndRemovesBeforeInner(t *testing.T) { + for _, tc := range []struct { + name string + mutate func(*http.Request) + wantOK bool + }{ + {name: "single alias", mutate: rekeyBaselineWireMarker, wantOK: true}, + {name: "duplicate aliases", mutate: func(req *http.Request) { + rekeyBaselineWireMarker(req) + req.Header[strings.ToUpper(baselineWireMarkerHeader)] = []string{baselineWireMarkerValue} + }}, + } { + t.Run(tc.name, func(t *testing.T) { + capture := &baselineWireCapture{ + stage: "acquire", setID: 7, requestIDs: []int64{41}, origin: "https://api.example:443", + runtimePathPrefix: "/tenant/v2", runtimePathPrefixSet: true, allowedHosts: []string{"api.example"}, + } + innerCalls := 0 + markerKeysAtInner := 0 + inner := drainRoundTripper(func(req *http.Request) (*http.Response, error) { + innerCalls++ + for key := range req.Header { + if strings.EqualFold(key, baselineWireMarkerHeader) { + markerKeysAtInner++ + } + } + return &http.Response{StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"count":1,"value":[41]}`)), Request: req}, nil + }) + transport := responseBudgetTransport{inner: baselineRequestMutationRoundTripper{inner: preservingContextRoundTripper{inner: baselineRequestCaptureTransport{inner: inner}}, mutate: tc.mutate}} + req, err := http.NewRequestWithContext(capture.context(context.Background()), http.MethodPost, "https://api.example/tenant/v2/_apis/runtime/runnerscalesets/7/acquirejobs?api-version=6.0-preview", strings.NewReader("[41]")) + if err != nil { + t.Fatal(err) + } + response, err := transport.RoundTrip(req) + if tc.wantOK { + if err != nil || response == nil || response.StatusCode != http.StatusOK || innerCalls != 1 || markerKeysAtInner != 0 || !capture.requestObserved() || !capture.observed() { + t.Fatalf("case-folded marker with preserved context = response %v err %v inner calls=%d marker keys=%d requestObserved=%v observed=%v, want one clean forward", response, err, innerCalls, markerKeysAtInner, capture.requestObserved(), capture.observed()) + } + return + } + if !errors.Is(err, ErrRemote) || response != nil || innerCalls != 0 || markerKeysAtInner != 0 || capture.requestObserved() || capture.observed() { + t.Fatalf("duplicate case-folded marker = response %v err %v inner calls=%d marker keys=%d requestObserved=%v observed=%v, want rejection before inner", response, err, innerCalls, markerKeysAtInner, capture.requestObserved(), capture.observed()) + } + }) + } +} From f5560ba950f77343e57034cc1cf85dc67f5ac922 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Sun, 13 Sep 2026 11:03:13 +0900 Subject: [PATCH 61/63] docs(g01): record folded marker evidence --- docs/evidence/g01-idle-drain.md | 86 +++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index d97f0ae..fcbd462 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2589,3 +2589,89 @@ documentation-only commit. Source rollback is recoverable with documentation append separately if needed. No live App/runner/canary test, Docker/Lima, Keychain, launchd or workflow replay was run; independent exact-head Codex review, CI and merge remain coordinator-owned. + +### Exact-head follow-up: case-folded baseline marker boundary + +Date: 2026-09-13. This correction started from exact PR #72 head +`5b24636c04b036da238c03efe4e27f983e138251` and addresses the two fresh Codex +P1 findings at [the replacement-context marker boundary](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998274209) +and [the context-preserved marker boundary](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998274210). +The findings identified that a request wrapper could rekey the private +baseline marker to a noncanonical case spelling, after which canonical-only +lookup and deletion could bypass the final marked-request boundary or expose +the marker to the physical transport. No Issue, Project, goal, dependency, +status or branch field was changed; no Codex review was requested, and no +live App/runner/canary operation, workflow replay, Docker/Lima, Keychain, +launchd, credential or cleanup operation was performed. + +#### Red-first reproduction + +The two case-folded marker regressions were added and run before the source +correction. This focused command exited 1 as expected: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaselineMarkedCaseFoldedMarkerWithReplacementContextRejectsBeforeInner|TestBaselineMarkedCaseFoldedMarkerPreservesContextAndRemovesBeforeInner)$' -count=1 -v -timeout=120s +``` + +The replacement-context control returned a successful synthetic response and +reached the inner transport instead of rejecting the marked request. The +context-preserved alias control also forwarded successfully and exposed one +folded marker key at the inner transport; its duplicate-alias control exposed +two folded marker keys instead of rejecting before forwarding. The regressions +retain only bounded inner-call, marker-key, status, error and observation +categories; no response body, authorization value, token, private path or raw +transport log is recorded. + +#### Minimal correction and green evidence + +The final baseline boundary now scans request-header map keys +case-insensitively, counts every matching marker key and aggregates their +values. A marker-bearing request without its capture context is rejected; +when the context is present, exactly one matching key and exactly one valid +marker value are required. Every case-folded marker key is removed before the +inner transport receives a valid request, while unmarked forwarding and the +existing direct capture test seam remain unchanged. The prior target, body, +authorization, Host/Opaque, capacity, origin, tenant-prefix, session identity +and one-shot checks continue to run after marker validation. + +The focused changed-boundary normal command exited 0 in 21.152s, and the +corresponding race command exited 0 in 33.676s with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestBaseline|TestMarkedRequest|TestUnmarked)' -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestBaseline|TestMarkedRequest|TestUnmarked)' -count=1 -timeout=360s +``` + +The full `livecanary` package normal run exited 0 in 26.815s and the full +race run exited 0 in 37.083s, with no race diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=360s +``` + +The changed-module vet and repository format checks exited 0; `git diff +--check` exited 0; the source/test diff secret/private-path scan exited 0 and +printed `diff secret/private-path scan passed`. The required two-module +offline gate exited 0 and printed `offline experiment checks passed: 2 +module(s)`: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go vet ./livecanary +cd ../.. +GOTOOLCHAIN=go1.26.8 bash scripts/gofmt.sh check +git diff --check +GOTOOLCHAIN=go1.26.8 bash scripts/check-offline-experiments.sh +``` + +The exact tested source/test head is +`0f36583535a7f26644e6b58815c8dd0b85b45b8b` (`fix(g01): bind folded baseline +wire markers`); the final source/evidence head is this source commit plus the +separate documentation-only append. Source rollback is recoverable with +`git revert --no-edit 0f36583535a7f26644e6b58815c8dd0b85b45b8b`; revert this +documentation append separately if needed. No live App/runner/canary evidence +is claimed; the unresolved live G01 gate, CI and merge remain coordinator-owned. From cd480d81312110e83404bb050254d14be84b54ff Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 16 Sep 2026 13:07:44 +0900 Subject: [PATCH 62/63] docs(g01): record exact-head P1 audit --- docs/evidence/g01-idle-drain.md | 80 +++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index fcbd462..a4d913b 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2675,3 +2675,83 @@ separate documentation-only append. Source rollback is recoverable with `git revert --no-edit 0f36583535a7f26644e6b58815c8dd0b85b45b8b`; revert this documentation append separately if needed. No live App/runner/canary evidence is claimed; the unresolved live G01 gate, CI and merge remain coordinator-owned. + +## Exact-head P1 audit: live API set for `f5560ba950f77343e57034cc1cf85dc67f5ac922` + +On 2026-09-16 UTC, the live GitHub API was re-queried before this evidence +update. The PR head was still +`f5560ba950f77343e57034cc1cf85dc67f5ac922`. Review comments whose +`commit_id` equals that SHA were treated as current even when their creation +date or wording made them look historical; older commit IDs were not +redispatched. The current-head P1 comments below are all addressed by the +existing source corrections and regressions already present at that SHA. The +live API uses `r3996893012` for the session-origin finding; the previously +written `r3976893012` spelling is not a live comment ID and is not treated as +authority. + +The evidence is bounded and offline. No request body, queue URL, token, JIT +value, raw SDK error, personal path or private log is included here. “Red” +means the historical or isolated pre-fix regression recorded in the linked +section; “green” names the current regression that was rerun on the exact +source head. A green result proves only the client-side invariant and does not +prove server receipt or an atomic remote drain. + +| Live finding | Red reproduction and current resolution evidence | +|---|---| +| [r3965776826](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3965776826) cancellation join | The cancellation regression previously allowed the listener goroutine to outlive the phase. `runDrainListener` now cancels, releases the held response, and joins on every context-exit path; `TestDrainRejectsEffectsAfterCancellationAndRecordsMarker`, `TestDrainCancellationAfterIntentRejectsEffect`, and `TestDrainCancellationBeforeSnapshotRecordsMarker` pass. | +| [r3966125441](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966125441) runner continuity | The runner replacement/missing-identity regression is retained in `TestDriverDrainThroughPinnedSDKAndPollHook`, `TestPinnedSDKDrainRejectsRunnerSnapshotOriginMismatchBeforeListener`, and the replay contract tests. Observed evidence requires two equal non-nil runner identities and retains uncertainty otherwise. | +| [r3966362990](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3966362990) durable drain uncertainty | The crash-before-final-observation history is rejected by `TestDrainPhaseStartRetainsCrashUncertainty`, `TestReplayDrainFenceRetainsInconclusiveOutcome`, and the FileJournal reopen tests. The phase fence is recorded before remote polling and is discharged only by one matching valid observation. | +| [r3967496305](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3967496305) finding ledger | This audit supplies the missing current-head URL ledger, red/green references, source-resolution evidence and the explicit live-operation gap. | +| [r3972526881](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972526881) embedded job body | The duplicate/case-folded embedded identity regression is covered by `TestPinnedSDKDrainRejectsAmbiguousEmbeddedJobIdentityBeforeEffects`. The bounded poll adapter strictly decodes the embedded message and compares it with the SDK object before `VerifyRun`, ACK or acquisition. | +| [r3972911049](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911049) withdrawn-poll body | The 202-with-message regression is covered by `TestPinnedSDKDrainRejectsWithdrawnPollBodyBeforeAbsent`; a lossy SDK nil cannot be classified as absent unless the bounded wire shape proves no message. | +| [r3972911056](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911056) acquisition response | Duplicate, missing, mismatched and case-folded count/value responses are covered by `TestPinnedSDKDrainAcquisitionRequiresStrictWireResponse`. The bounded adapter requires strict count/value facts before acquisition success. | +| [r3972911063](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911063) scale-set snapshots | Duplicate/case-folded snapshot fields are covered by `TestPinnedSDKDrainSnapshotsRequireStrictWireFacts` and `TestPinnedSDKDrainRejectsAmbiguousRunnerSnapshot`; set and runner evidence is bounded, strict and origin/prefix-bound before it is journaled. | +| [r3972911082](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3972911082) workflow-run authorization | `TestPinnedSDKDrainVerifyRunRejectsAmbiguousWireFieldsBeforeEffects` covers duplicate/case-folded authorization fields. `VerifyRun` uses the strict bounded observation adapter and cannot authorize effects from ambiguous JSON. | +| [r3973406553](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973406553) drain session response | `TestPinnedSDKDrainRejectsAmbiguousSessionResponse` covers duplicate/case-folded session identity, queue, token and statistics fields. `OpenDrainSession` captures and strictly validates the response before exposing a session to the listener. | +| [r3973406571](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973406571) physical ACK | `TestPinnedSDKDrainBindsACKToPhysicalDelete` and the marked ACK mismatch/cardinality tests require the exact queue origin/path, cursor, set/session identity and one physical DELETE before ACK success is journaled. | +| [r3973406578](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3973406578) poll read completion | `TestPinnedSDKDrainRejectsNonEOFPollReadError` and `TestPinnedSDKDrainRejectsPollCloseErrorBeforeEffects` force poll wire facts unknown on any non-EOF read or close failure before later effects. | +| [r3974007586](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3974007586) physical session close | `TestPinnedSDKDrainBindsSessionCloseToPhysicalDelete` and `TestBaselineMarkedSessionCloseMismatchStopsBeforeInner` require the captured session/set route, origin, tenant prefix, authorization and one-shot DELETE plus a clean 204 before clearing the session fence. | +| [r3974562078](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3974562078) session-open target | `TestBaselineSessionOpenTargetMismatchStopsBeforeInner` exercises host, route, set, method and query mutations. Marked non-bootstrap session-open candidates are rejected before the inner transport; the narrow registration bootstrap allowlist remains explicit. | +| [r3976171401](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976171401) acquisition/session origin | `TestBaselineAcquireTargetRequiresCapturedSessionOrigin`, `TestBaselineAcquireOriginMismatchStopsBeforeInner`, and `TestPinnedSDKDrainBindsAcquireToPhysicalRequestBody` require acquisition to use the exact captured session-open origin, not merely an approved-host member. | +| [r3976536000](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976536000) session-open body | `TestBaselineSessionOpenBodyMustMatchOwnerBeforeInner` rejects malformed, duplicate, case-folded and wrong-owner bounded bodies before forwarding; no usable session identity is persisted on failure. | +| [r3976535985](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976535985) acquisition pre-effect fence | `TestBaselineAcquireTargetMismatchStopsBeforeInner`, `TestBaselineAcquireTargetIsActionsOnly`, and the pinned-SDK target mutation matrix reject queue-shaped, wrong-host, wrong-set/path and wrong-method requests before the inner transport. | +| [r3976535994](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976535994) snapshot origin | `TestBaselineSnapshotRequestsRequireExactOriginBeforeInner` and the pinned snapshot-origin regressions require before/after set and runner reads to use the captured origin and tenant prefix. | +| [r3976536008](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3976536008) finding URLs | The linked correction section records the red mutation matrix, green normal/race commands and rollback scope for the acquisition-origin, snapshot-origin and session-open-body findings. | +| [r3996893012](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996893012) drain-session origin | `TestPinnedSDKDrainRejectsSessionOriginMismatchBeforeListener` rejects a session opened on a different origin than the before snapshot before any listener poll. | +| [r3996992592](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3996992592) duplicate session-open | `TestBaselineSessionOpenRejectsDuplicateMarkedPOSTBeforeInner` reserves the marked session-open one-shot before forwarding and rejects every duplicate. | +| [r3997097883](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997097883) ACK target | `TestBaselineMarkedACKMismatchStopsBeforeInner` and `TestBaselineMarkedACKRequiresIdentityAndOneShotCardinality` reject rewritten queue/message targets before the inner transport. | +| [r3997211059](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997211059) Host authority | `TestMarkedRequestHostOverrideStopsBeforeInner` rejects a non-empty `Request.Host` unless it canonically equals `URL.Host`; `TestMarkedRequestHostMatchingURLHostPreservesForwarding` preserves the valid control, including the marked-poll boundary. | +| [r3997312693](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997312693) opaque target | `TestMarkedRequestOpaqueStopsBeforeInner` and `TestMarkedPollOpaqueStopsBeforeInner` reject every non-empty marked `URL.Opaque` before the final physical transport; unmarked forwarding remains unchanged. | +| [r3997904130](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997904130) runner response completion | `TestGuardBaselineResponseRejectsRunnerFactsWhenBodyCloseFails` invalidates runner facts on response body close failure before continuity or observation evidence can use them. | +| [r3997904133](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3997904133) marker propagation | `TestBaselineMarkedBoundariesRejectReplacementContextBeforeInner`, `TestBaselineWireMarkerIsRemovedBeforeInner`, and the case-folded marker regressions require exactly one valid marker at the final boundary and remove it before physical forwarding. | +| [r3998020455](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998020455) successful DELETE completion | `TestGuardBaselineResponseRejectsTerminalCloseWhenBodyCloseFails` invalidates expected-204 ACK/session-close captures on body-close failure; status alone cannot publish success. | +| [r3998106048](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998106048) JIT tuple | `TestBaselineMarkedJITRequestRejectsPhysicalTupleMutationBeforeInner` rejects JIT origin, tenant-prefix and strict body mutations before forwarding; valid marked and unmarked controls remain green. | +| [r3998189095](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r3998189095) verification completion | This is the same close-completion boundary as issue comment [5649825382](https://github.com/1XP-AI/gh-runnerd/pull/72#issuecomment-5649825382). `TestPinnedSDKDrainRejectsVerifyRunCloseErrorBeforeEffects` rejects a matching workflow-run body whose `Close` fails before ACK/acquisition. | + +The current source audit was rerun with these offline commands on the exact +source head before this documentation-only append: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^(TestDrain|TestReplay|TestBaseline|TestMarked|TestUnmarked|TestPinnedSDKDrain|TestGuardBaselineResponseRejects|TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine|TestValidatePairedApprovalsAcceptsDrainVerification)$' -count=1 -timeout=300s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^(TestDrain|TestReplay|TestBaseline|TestMarked|TestUnmarked|TestPinnedSDKDrain|TestGuardBaselineResponseRejects|TestDriverRoutesDrainBeforeNoWorkerStatisticsQuarantine|TestValidatePairedApprovalsAcceptsDrainVerification)$' -count=1 -timeout=360s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=360s +``` + +All three commands exited 0 (focused normal 0.473s, focused race 1.334s, +package normal 29.423s). The package race run also exited 0 in 55.684s with no +race diagnostics. These additional checks exited 0: `go vet ./livecanary`, +`bash scripts/gofmt.sh check`, `git diff --check`, and +`bash scripts/check-offline-experiments.sh` (which printed +`offline experiment checks passed: 2 module(s)`). The changed documentation +diff secret/private-path scan also exited 0 and printed +`diff secret/private-path scan passed`. + +No live GitHub Actions call, runner, Scale Set, session, JIT, workflow, +Docker/Lima, Keychain, launchd, credential or cleanup operation was run. The +explicit unresolved gap is therefore live validation and server receipt: these +client-side tests cannot establish an atomic remote drain, and the independent +NO-GO safety findings recorded in +[issue-comment 5680387706](https://github.com/1XP-AI/gh-runnerd/pull/72#issuecomment-5680387706) +remain quarantined in stacked PR #81 rather than silently claimed as fixed by +this offline PR. From 4d98f68361763c7f0f84ec36f87238f16df8a235 Mon Sep 17 00:00:00 2001 From: JinWoo Lee Date: Wed, 16 Sep 2026 13:42:35 +0900 Subject: [PATCH 63/63] fix(g01): reject preflight response close failures --- docs/evidence/g01-idle-drain.md | 70 +++++++++++++++++++ experiments/g01-scaleset/livecanary/sdk.go | 7 +- .../g01-scaleset/livecanary/sdk_test.go | 58 +++++++++++++++ 3 files changed, 132 insertions(+), 3 deletions(-) diff --git a/docs/evidence/g01-idle-drain.md b/docs/evidence/g01-idle-drain.md index a4d913b..e47a7e5 100644 --- a/docs/evidence/g01-idle-drain.md +++ b/docs/evidence/g01-idle-drain.md @@ -2755,3 +2755,73 @@ NO-GO safety findings recorded in [issue-comment 5680387706](https://github.com/1XP-AI/gh-runnerd/pull/72#issuecomment-5680387706) remain quarantined in stacked PR #81 rather than silently claimed as fixed by this offline PR. + +### Exact-head follow-up: reject incomplete preflight responses before draining + +Date: 2026-09-16. This correction starts from exact PR #72 head +`cd480d81312110e83404bb050254d14be84b54ff` and addresses the Codex P1 finding +at [discussion_r4022441222](https://github.com/1XP-AI/gh-runnerd/pull/72#discussion_r4022441222). +The finding identified that `SDKAPI.get` deferred `response.Body.Close` and +could authorize preflight from complete-looking JSON even when close reported +an error. `SDKAPI.get` has two callers: the preflight authority checks and the +legacy inventory reader; both retain their existing fixed error categories. +No Issue, Project, goal, dependency, live-operation or runner state was +changed. + +#### Red-first reproduction + +The regression used the existing offline loopback HTTP fixture. Before the +source correction, the first preflight response returned complete valid JSON +but its body returned `io.ErrClosedPipe` from `Close`; `Run("drain")` proceeded +past preflight and attempted the first drain snapshot. This focused command +exited 1 as expected: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPreflightRejectsCompleteResponseWhenBodyCloseFailsBeforeDrain$' -count=1 -v -timeout=60s +``` + +The bounded failure was: + +```text +sdk_test.go:134: preflight body close error = state uncertain; quarantine and inspect only, want approval rejection +``` + +The regression retains only the fixed approval/quarantine categories, a drain +snapshot call count and journal length; it records no response body, token, +URL, raw SDK error or private log. + +#### Minimal correction and green evidence + +`SDKAPI.get` now explicitly closes successful responses after the bounded +`io.ReadAll` and rejects any read or close completion error before JSON can +authorize a caller. Non-200 responses still close and return the existing +opaque `ErrRemote`; the 1 MiB response limit and JSON validation are unchanged. +The regression passed normally in 0.133s and under race in 1.328s with no race +diagnostics: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -run '^TestPreflightRejectsCompleteResponseWhenBodyCloseFailsBeforeDrain$' -count=1 -v -timeout=60s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -run '^TestPreflightRejectsCompleteResponseWhenBodyCloseFailsBeforeDrain$' -count=1 -timeout=60s +``` + +The full `livecanary` package passed normally in 25.121s and under race in +41.019s: + +```text +cd experiments/g01-scaleset +GOWORK=off GOTOOLCHAIN=go1.26.8 go test ./livecanary -count=1 -timeout=360s +GOWORK=off GOTOOLCHAIN=go1.26.8 go test -race ./livecanary -count=1 -timeout=420s +``` + +The source/test correction was first captured in focused commit +`21b67c5eeaf7e087a2b76a1c147105276b65b5fb`; the final single +implementation/docs commit is the exact branch `HEAD` reported at delivery +(verify with `git rev-parse HEAD`), and its rollback is +`git revert --no-edit HEAD`. No live GitHub call, App enrollment, runner/Scale +Set operation, workflow replay, Docker/Lima, Keychain or launchd operation was +run. The live-validation gap is explicitly retained: these offline tests prove +only the client-side fail-closed boundary and cannot prove remote server +receipt or an atomic live drain; independent exact-head Codex review, CI and +maintainer-authorized live validation remain coordinator-owned. diff --git a/experiments/g01-scaleset/livecanary/sdk.go b/experiments/g01-scaleset/livecanary/sdk.go index cbd1d31..c9191a0 100644 --- a/experiments/g01-scaleset/livecanary/sdk.go +++ b/experiments/g01-scaleset/livecanary/sdk.go @@ -139,12 +139,13 @@ func (a *SDKAPI) get(ctx context.Context, path, token string, target any) error if err != nil { return ErrRemote } - defer resp.Body.Close() if resp.StatusCode != http.StatusOK { + _ = resp.Body.Close() return ErrRemote } - data, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20+1)) - if err != nil || len(data) > 1<<20 || json.Unmarshal(data, target) != nil { + data, readErr := io.ReadAll(io.LimitReader(resp.Body, 1<<20+1)) + closeErr := resp.Body.Close() + if readErr != nil || closeErr != nil || len(data) > 1<<20 || json.Unmarshal(data, target) != nil { return ErrRemote } return nil diff --git a/experiments/g01-scaleset/livecanary/sdk_test.go b/experiments/g01-scaleset/livecanary/sdk_test.go index 6229f6a..4b2ac4b 100644 --- a/experiments/g01-scaleset/livecanary/sdk_test.go +++ b/experiments/g01-scaleset/livecanary/sdk_test.go @@ -3,6 +3,8 @@ package livecanary import ( "context" "encoding/json" + "errors" + "io" "net/http" "net/http/httptest" "strings" @@ -83,6 +85,62 @@ func TestAuthoritySplitAndPolicyRejection(t *testing.T) { } } +type preflightDrainProbe struct { + *SDKAPI + drainCalls int +} + +func (p *preflightDrainProbe) drainGetScaleSet(context.Context, int, *baselineWireCapture) (*scaleset.RunnerScaleSet, error) { + p.drainCalls++ + return nil, ErrRemote +} + +func TestPreflightRejectsCompleteResponseWhenBodyCloseFailsBeforeDrain(t *testing.T) { + a := approval() + a.Phases = append(a.Phases, "drain") + c := credentials(a) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + repo := map[string]any{"id": a.RepositoryID, "full_name": a.Organization + "/" + a.Repository, "private": true, "fork": false} + var body any + switch { + case r.URL.Path == "/installation/repositories": + body = map[string]any{"total_count": 1, "repositories": []any{repo}} + case strings.HasSuffix(r.URL.Path, "/repositories"): + body = map[string]any{"total_count": 1, "repositories": []any{repo}} + case strings.Contains(r.URL.Path, "/runner-groups/"): + body = map[string]any{"id": a.RunnerGroupID, "visibility": "selected", "default": false, "allows_public_repositories": false, "inherited": false} + default: + body = repo + } + if r.Method != http.MethodGet || r.Header.Get("Authorization") != "Bearer "+c.InstallationToken { + t.Error("authority crossed endpoint boundary or preflight wrote") + } + _ = json.NewEncoder(w).Encode(body) + })) + defer server.Close() + + client := server.Client() + client.Transport = sdkResponseBodyFaultRoundTripper{ + inner: client.Transport, + path: "/installation/repositories", + closeErr: io.ErrClosedPipe, + } + api := &preflightDrainProbe{ + SDKAPI: &SDKAPI{rest: client, baseURL: server.URL, approval: a, credentials: c}, + } + j := &memoryJournal{events: append([]Event(nil), drainReplayPrefix()[:3]...)} + d := Driver{Approval: a, Journal: j, API: api} + if err := d.Run(context.Background(), "drain"); !errors.Is(err, ErrApproval) { + t.Fatalf("preflight body close error = %v, want approval rejection", err) + } + if api.drainCalls != 0 { + t.Fatalf("body close error entered drain snapshot: calls=%d", api.drainCalls) + } + if len(j.Events()) != 3 { + t.Fatalf("body close error appended drain phase: events=%d, want 3", len(j.Events())) + } +} + func TestCredentialAttestationMismatchAndExpiredTokenRejected(t *testing.T) { for _, fault := range []string{"app", "installation", "org", "permission", "expiry", "same-token"} { t.Run(fault, func(t *testing.T) {