Reconcile orphan queue intents and make member maintenance preserve workers

cmd/gha-fleet/main.go

@@ -1713,22 +1713,6 @@ func (c drainClient) GetInstances(kind api.InstanceType) ([]api.Instance, error)
 	return c.incus.GetInstances(kind)
 }
 
-func (c drainClient) DeleteInstance(name string) error {
-	// Incus refuses to delete a running instance, and a warm occupant is
-	// running by definition. Force-stop first; if the stop fails because the
-	// instance is already stopped or already gone, the delete below is the
-	// call that decides.
-	stop, err := c.incus.UpdateInstanceState(name, api.InstanceStatePut{Action: "stop", Force: true, Timeout: -1}, "")
-	if err == nil {
-		_ = stop.Wait()
-	}
-	op, err := c.incus.DeleteInstance(name)
-	if err != nil {
-		return err
-	}
-	return op.Wait()
-}
-
 // pressureGate closes and reopens the member's gate through the publisher that
 // owns it. Writing scheduler.instance directly is overwritten within one cycle.
 type pressureGate struct {
@@ -1961,6 +1945,7 @@ func runDrainMember(args []string, stdout, stderr io.Writer) int {
 	timerUnit := flags.String("timer-unit", memberdrain.DefaultTimerUnit, "the timer that owns this member's gate")
 	reason := flags.String("reason", "", "why the member is being taken out of service, published as the gate's close reason")
 	restore := flags.Bool("restore", false, "hand the member back: republish from live pressure and start the timer")
+	fenceOnly := flags.Bool("fence-only", false, "close placement and report occupants without waiting for a drained member")
 	timeout := flags.Duration("timeout", memberdrain.DefaultTimeout, "how long to wait for running jobs to finish")
 	poll := flags.Duration("poll", memberdrain.DefaultPoll, "how often to re-read what the member is carrying")
 	apply := flags.Bool("apply", false, "stop the timer and publish, rather than reporting what would happen")
@@ -1971,8 +1956,8 @@ func runDrainMember(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintln(stderr, "gha-fleet: drain-member requires --config and no positional arguments")
 		return 2
 	}
-	if *restore && *reason != "" {
-		fmt.Fprintln(stderr, "gha-fleet: drain-member --restore takes no --reason")
+	if *restore && (*reason != "" || *fenceOnly) {
+		fmt.Fprintln(stderr, "gha-fleet: drain-member --restore takes neither --reason nor --fence-only")
 		return 2
 	}
 	cfg, err := config.Load(*configPath)
@@ -2015,7 +2000,7 @@ func runDrainMember(args []string, stdout, stderr io.Writer) int {
 	}
 	options := memberdrain.Options{
 		MemberName: cfg.Incus.Cluster.MemberName, Reason: *reason,
-		TimerUnit: *timerUnit, Timeout: *timeout, Poll: *poll, Apply: *apply,
+		TimerUnit: *timerUnit, Timeout: *timeout, Poll: *poll, Apply: *apply, FenceOnly: *fenceOnly,
 	}
 	var result memberdrain.Result
 	if *restore {

config/example-runner-1.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.94
+  manager_version: v0.2.1-nddev.95
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.132
+  provider_version: v0.1.5-nddev.133
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-2.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.94
+  manager_version: v0.2.1-nddev.95
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.132
+  provider_version: v0.1.5-nddev.133
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-3.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.94
+  manager_version: v0.2.1-nddev.95
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.132
+  provider_version: v0.1.5-nddev.133
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-4.yaml

@@ -6,10 +6,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.94
+  manager_version: v0.2.1-nddev.95
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.132
+  provider_version: v0.1.5-nddev.133
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-services.yaml

@@ -24,10 +24,10 @@ platform:
 
 control_plane:
   manager: garm
-  manager_version: v0.2.1-nddev.94
+  manager_version: v0.2.1-nddev.95
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.132
+  provider_version: v0.1.5-nddev.133
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/garm-derivative.yaml

@@ -1,6 +1,6 @@
 schema_version: 1
 artifact: garm
-derivative_version: v0.2.1-nddev.94
+derivative_version: v0.2.1-nddev.95
 upstream:
   repository: https://github.com/cloudbase/garm
   release: v0.2.1
@@ -109,9 +109,15 @@ patches:
   - path: third_party/garm/patches/0034-bump-golang-x-text-v0.39.0.patch
     sha256: 2d23c290fe462553607739afeccca2fe1dcc13d2352e6324464f35fa3ffb6bc2
     purpose: Upgrade the one source-reachable advisory, golang.org/x/text, to v0.39.0. x/mod v0.37.0 and x/sync v0.21.0 are required transitives of that release. OpenPGP is not imported.
+  - path: third_party/garm/patches/0035-reconcile-journal-without-db-row.patch
+    sha256: 1544734007932649e58af7d0b35d83d4e06aeb10a1d1b03d049272953ca0e7b9
+    purpose: Feed repository-bound journal entries into the periodic exact GitHub verifier even after DB cleanup; preserve unknown, running, wrong-scope and ambiguous identities, keep bounded per-GUID retry state, and distinguish a complete absent GUID from an incomplete response.
+  - path: third_party/garm/patches/0036-refresh-vendored-go-dependencies.patch
+    sha256: 2417a1ff7ec61f3e03311ece1ee32d38ecb9f4e1e4efb2d43ab48e10d371f509
+    purpose: Update the vendored Go dependency closure and language/toolchain pins to reviewed compatible current releases; preserve offline test/build reproducibility and remove obsolete advisory-bearing x packages.
 overlays:
   - path: third_party/garm/overlay/workers/scaleset/queue_intent.go
-    sha256: c15fb2a6b82ad6cc6b79708d1608aa7ee026bbf2a3c17f247775dfd8e51d2735
+    sha256: 4b1b9e926a7492423c5ef118ec2a8c0793211e622d16ef37a94f8b2b75604915
     purpose: Fsync-backed central queue admission that excludes terminal capacity, gives completion precedence in fast start/complete batches, requests fresh demand when persisted desired is zero, demotes an expired never-started assigned waiter to queued instead of deleting it, refreshes queued TTL up to the execution horizon, FIFO-orders by FirstQueuedAt, treats listener JobCompleted as a delivery-reservation end rather than a REST workflow-job terminal, does not alias a later same-run/name GUID onto that waiter, does not delete or rename an assigned waiter because an unrelated JobStarted arrived, and yields request-less occupancy only when a dispatchable JobAvailable would actually fit after that yield.
   - path: third_party/garm/overlay/workers/scaleset/queue_intent_test.go
     sha256: abafa5100ee6bbf8ee1dd94d79e34222a3fbc5ed628c022487137896cd1fc5dd
@@ -143,9 +149,21 @@ overlays:
   - path: third_party/garm/overlay/runner/pool/authoritative_reconcile_backoff_test.go
     sha256: d0c8f58b0fe98f5aa496dc161d8064b4029f430b8836c1d37f4429f27a7de620
     purpose: Prove GitHub 403 and 429 establish the 15-minute access-refusal backoff, while 404 and 500 retain ordinary per-job retry without extending that backoff. Gate unit tests, not Worker/DB/HTTP integration.
+  - path: third_party/garm/overlay/workers/scaleset/queue_reconciliation.go
+    sha256: 46bf21b502904d853ef4e4a079283901a39f395c22367237e79d516b094aec58
+    purpose: Read bounded non-running repository-bound journal candidates without mutating TTLs, capacity, execution or terminal lineage; retain exact GUID, owner, repository, run and scale-set identities.
+  - path: third_party/garm/overlay/workers/scaleset/queue_reconciliation_test.go
+    sha256: d11766b2efc01f08a300c48408a1d150d6730160efebcce7feb26f0380f313bb
+    purpose: Prove run ID survives new and already-bound authoritative rehydration, conflicting identities and repeats cannot renew state, and candidate reads preserve running, acquired, claimed, foreign, fresh and terminal work.
+  - path: third_party/garm/overlay/runner/pool/queue_reconciliation.go
+    sha256: 118bd4e553ebe8bc8f9e5a90b039bb36e36b91e27a91995dde213fe5b6225b2f
+    purpose: Reconcile at most two journal candidates per pass with a five-minute per-GUID budget. Missing legacy run identity is discovered only in a complete ten-run window spanning thirty minutes before first queue observation plus one minute, then independently verified by exact GitHub GUID/run/attempt/check URL/SHA/scale-set binding.
+  - path: third_party/garm/overlay/runner/pool/queue_reconciliation_test.go
+    sha256: b5f5e3234984fde3e5afa99a77d848b079f98c73750fe19638526788b52f89fa
+    purpose: "Exercise the real periodic entry point with SQLite and the native durable journal: remove an exact completed orphan without any DB row, preserve a running sibling, persist a terminal receipt across restart, reject wrong scale set, access refusal, incomplete/ambiguous discovery, and rate-limit retained in-progress work."
 build:
-  container_image: docker.io/library/golang@sha256:116d58cbd88c1297624acc6e967a060012422bacf9930927e23fb719189c6f36
-  go_version: go1.26.6
+  container_image: docker.io/library/golang@sha256:6860a573025fcd35aba4f1b23e858b1c1dcd425d53fa73562e169a6ab1c2339a
+  go_version: go1.27.1
   cgo_enabled: true
   target_os: linux
   target_arch: amd64
@@ -157,7 +175,7 @@ build:
     - sqlite_omit_load_extension
   reproducible_rebuilds: 2
   maximum_required_glibc: "2.34"
-  binary_sha256: 75ec2a259092859bbb42b448f99b549bc33606649b4c953bf4292b006c7652a9
+  binary_sha256: 1b4014dbf23eaeffd7706772ce7d080beeb5c9af82f771021aa5c29a9ad27abd
 runtime_contract:
   queue_intent_schema_version: 6
   event_driven_scale_set_wake: true

config/provider-derivative.yaml

@@ -16,7 +16,7 @@ artifact: garm-provider-incus
 # state all move together, because all three derive from here. A provider change
 # that does not bump it ships under the previous version, which is exactly how
 # runner-1 and runner-2 diverged.
-derivative_version: v0.1.5-nddev.132
+derivative_version: v0.1.5-nddev.133
 
 # The external-provider protocol GARM speaks to this binary. It moves on its own
 # schedule -- a provider release does not imply an interface release -- so it is
@@ -37,8 +37,8 @@ runtime:
   queue_intent_schema_version: 6
 
 build:
-  source_commit: 7a82d685b1b658f6aa66a8967ae91ed2b92df71b
-  binary_sha256: 5769660e5d6f2f2efc0925fe55ed5987110f46e60a8a9e58aac061c12189961c
+  source_commit: 3d8b8983ce5128baf922d774f1e91dbfabd8d3b4
+  binary_sha256: 515bf60fa275849d1d9ded006a491d286c194854c1eae743475c53f8218dbf01
   go_version: go1.27.1
   cgo_enabled: false
   target_os: linux

docs/maintenance-windows.md

@@ -13,15 +13,22 @@ one arc and the first live windows found what the fakes could not.
   timer running, so every tick republishes a fresh closed state carrying
   `drained: <reason>` — the staleness alert stays armed and silent for the
   right reason, and the drain survives a reboot;
-- it **recycles warm occupants** (force-stop, then delete: Incus refuses to
-  delete a running instance) instead of waiting out disposable capacity —
-  and since provider v0.1.5-nddev.107 the warm reconciler also recycles
-  outdated-identity or outdated-image instances on its own timer, so a
-  stale-stamp sweep after maintenance is no longer a manual step;
-- it **waits for real jobs** to finish and never aborts one;
+- it waits for **all running occupants**, including containers named `warm-*`,
+  and never stops or deletes them. A claimed warm retains its name, and even
+  a warm-looking metadata snapshot does not grant the member the provider's
+  claim lock. Unassigned warm retirement belongs to the provider's
+  `warm-drain` operation with its documented manager/claim preconditions;
 - `--restore --apply` clears the marker, republishes from live pressure, and
   heals members drained by older controllers.
 
+For a manager-only maintenance window, add `--fence-only`. The command closes
+placement and reports existing occupants immediately; occupied members are
+not reported as drained. This mode preserves warm capacity and running work.
+A manager replacement must still prove its own provider/worker quiescence
+before stopping the process. Full member drains, including slab healing, wait
+for warm retirement to be coordinated by its owner instead of force-stopping
+an instance. Restore the marker if a maintenance window is abandoned.
+
 The reason string travels: the gate publishes it into the cluster member
 config, and the observer reads it there to tell maintenance from an incident.
 

docs/queue-journal-reconciliation.md

@@ -0,0 +1,35 @@
+# Queue journal reconciliation
+
+A durable queue intent and a GARM workflow-job row have different lifetimes.
+Normal webhook cleanup can remove a DB row before the scale-set listener
+settles its journal entry. Absence of that row is not proof of completion.
+
+GARM now reads the journal independently during periodic reconciliation.
+Candidates are old, repository-bound queued/assigned entries without a runner
+or acquire request. Running/acquired work, fresh entries, foreign owners and
+active terminal receipts are excluded. The read never renews TTLs or changes
+resource ownership. Authoritative rehydration preserves workflow run identity,
+including on an already repository-bound entry, and refuses a conflicting run.
+
+The exact GitHub verifier binds GUID, owner/repository, current run attempt,
+check_run_url, GitHub Actions producer, source SHA, labels and the expected
+scale-set ID before releasing an intent. Its normal journal transaction writes
+a terminal receipt and preserves unrelated execution. No job is cancelled or
+rerun, and no outcome is inferred from expiry, DB absence or a name match.
+
+Legacy entries without run identity may use a complete window of at most ten
+runs: thirty minutes before the first queue observation through one minute
+after it. Every returned run and check-page boundary is validated; absent,
+incomplete or duplicate GUID evidence remains unproven. Two candidates per
+pass and a five-minute per-GUID budget bound this exceptional search. A
+successful DB reconciliation that retains an in-progress journal entry keeps
+that budget, preventing a repeated-request loop.
+
+Regression tests exercise the periodic entry point with SQLite and the native
+private journal, including missing DB rows, preserved running siblings,
+restart/idempotence, identity conflict, access refusal, incomplete discovery,
+duplicate GUIDs and wrong scale sets. Consumers record actual runtime adoption
+and observed terminal convergence separately from these tests.
+
+API identity fields: [workflow jobs](https://docs.github.com/en/rest/actions/workflow-jobs)
+and [workflow runs](https://docs.github.com/en/rest/actions/workflow-runs).

go.mod

@@ -89,7 +89,7 @@ require (
 	go.opentelemetry.io/proto/otlp v1.11.0 // indirect
 	go.yaml.in/yaml/v3 v3.0.5 // indirect
 	go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
-	golang.org/x/net v0.58.0 // indirect
+	golang.org/x/net v0.59.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
 	golang.org/x/term v0.46.0 // indirect
 	golang.org/x/text v0.42.0 // indirect

go.sum

@@ -262,8 +262,8 @@ golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZ
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.58.0 h1:ynWG7rqYi4ccpTEuPZ2QGWHktVEM9DMCj9yzDE0Q7To=
-golang.org/x/net v0.58.0/go.mod h1:YwCddHnFlT7eLQqVprV19OnhLGtc5xOKgE0RyqgfWAU=
+golang.org/x/net v0.59.0 h1:5zfYln+w5XCxwrnMMJPufRgNoXEaGxl0wo5GqPXyues=
+golang.org/x/net v0.59.0/go.mod h1:2DA/G1UfVbCpQPeWTmMPGY7Cs2PkBkwu743bVX5PIVg=
 golang.org/x/oauth2 v0.36.0 h1:peZ/1z27fi9hUOFCAZaHyrpWG5lwe0RJEEEeH0ThlIs=
 golang.org/x/oauth2 v0.36.0/go.mod h1:YDBUJMTkDnJS+A4BP4eZBjCqtokkg1hODuPjwiGPO7Q=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

internal/memberdrain/drain.go

@@ -17,18 +17,16 @@
 //
 // A drain never stops a running worker. It closes the gate so no new work is
 // placed, then waits for the jobs already there to finish on their own --
-// except warm instances, which are ready-unregistered by definition, hold
-// nobody's job, and are recycled rather than waited out: one held a reboot
-// hostage for a full forty-five-minute timeout. If real jobs outlast the
-// deadline the drain reports that it is still occupied and by what; it does
-// not decide to end someone's build.
+// including containers whose names begin with warm-. A consumed warm keeps
+// its name, and this member-local client does not own the provider claim lock.
+// Warm retirement belongs to the provider's journal-aware maintenance path.
+// Fence-only mode closes placement without declaring the member drained.
 package memberdrain
 
 import (
 	"context"
 	"fmt"
 	"sort"
-	"strings"
 	"time"
 
 	"github.com/lxc/incus/v7/shared/api"
@@ -39,9 +37,6 @@ import (
 // cluster's containers.
 type Client interface {
 	GetInstances(api.InstanceType) ([]api.Instance, error)
-	// DeleteInstance removes one instance. The drain uses it only for warm
-	// instances, which carry no job by construction.
-	DeleteInstance(name string) error
 }
 
 // Units is the systemd control a drain needs. Stopping the pressure timer is
@@ -89,6 +84,7 @@ type Options struct {
 	Timeout    time.Duration
 	Poll       time.Duration
 	Apply      bool
+	FenceOnly  bool
 }
 
 // Occupant is one instance still held by the member.
@@ -120,6 +116,7 @@ type Result struct {
 	Drained           bool       `json:"drained"`
 	TimedOut          bool       `json:"timed_out"`
 	Applied           bool       `json:"applied"`
+	FenceOnly         bool       `json:"fence_only,omitempty"`
 }
 
 const (
@@ -225,7 +222,10 @@ func Drain(ctx context.Context, deps Deps, options Options) (Result, error) {
 	}
 	result := Result{
 		MemberName: options.MemberName, Action: "drain", Reason: options.Reason,
-		TimerUnit: options.TimerUnit, Applied: options.Apply,
+		TimerUnit: options.TimerUnit, Applied: options.Apply, FenceOnly: options.FenceOnly,
+	}
+	if options.FenceOnly {
+		result.Action = "fence"
 	}
 	if !options.Apply {
 		occupants, err := Occupancy(deps.Client, options.MemberName)
@@ -263,28 +263,15 @@ func Drain(ctx context.Context, deps Deps, options Options) (Result, error) {
 		if err != nil {
 			return Result{}, err
 		}
-		// A warm instance is ready-unregistered by definition: it holds no
-		// job, and the maintainer refills it on an open member. Waiting for
-		// one is waiting for nothing -- one held a reboot hostage for the
-		// full timeout -- so warm occupants are recycled, not waited out.
-		remaining := occupants[:0]
-		for _, occupant := range occupants {
-			if strings.HasPrefix(occupant.Name, "warm-") {
-				if err := deps.Client.DeleteInstance(occupant.Name); err != nil {
-					return Result{}, fmt.Errorf("recycle warm occupant %s: %w", occupant.Name, err)
-				}
-				result.RecycledWarm = append(result.RecycledWarm, occupant.Name)
-				continue
-			}
-			remaining = append(remaining, occupant)
-		}
-		occupants = remaining
 		result.Occupants = occupants
 		result.WaitedSecs = int(deps.now().Sub(started) / time.Second)
 		if len(occupants) == 0 {
 			result.Drained = true
 			return result, nil
 		}
+		if options.FenceOnly {
+			return result, nil
+		}
 		if !deps.now().Before(deadline) {
 			// The member is closed and no new work lands on it. What is left is
 			// somebody's build, and ending it is not this command's decision.