fix: publish provider 132 with coherent inventory refresh

.gds/bundle.lock.yaml

@@ -2,16 +2,16 @@
 schema_version: 1
 
 bundle:
-  version: "0.8.0"
-  release_sequence: 55
+  version: "0.9.4"
+  release_sequence: 61
   channel: "stable"
-  source_tree_digest: "sha256:059be9ccd7b94dbb2ed1d00de45ea0f33de430325aa15f0646c04eb35fd10249"
-  digest: "sha256:e582619f2c9bf192b57b761418d52afe520630a074cbdae3ae9199223c33c93e"
-  attestation_identity_digest: "sha256:e1b787c63de916f750e2cbcbf664967669024e1062964515f612b2c826c9e10d"
+  source_tree_digest: "sha256:7539972f647cb41bde607b2df27ba7edc6f38944cd03bd3998e0399feb81fc38"
+  digest: "sha256:d1952b8e599e5b71b7a4e37614817b6ec30081003f1380a906b6a62defe2a332"
+  attestation_identity_digest: "sha256:bae54be9c903b0ec1d356ce3ae4b54b4226f0ae978c3c8d544fccde2d17bd771"
 
 projection:
-  input_digest: "sha256:16d00b8c8cfe471cc00e14ab983f28cbed63448fc13f2f7f35977d827a172f11"
-  output_digest: "sha256:6df8425d9b3ebe48a39fad7d21493444364c697256721cd692ea35e1658abf0e"
+  input_digest: "sha256:ac8c5e5d8fa76f0114698ebc2a8653a05f06bdf24ca3eea17c87f4e701657099"
+  output_digest: "sha256:766c85527ee2ce108bd40901d90751dd048d201d3dee72721aef872e0fd78ac6"
   files:
     - path: ".gds/compiled-policy.json"
-      digest: "sha256:6f72b631e71dfc641101dad2b84e9a4bd6c42281f9c87e168d883aa85d11c20d"
+      digest: "sha256:71e0f6cb2a4e0e3e34968e900a5f2969b372ceaab6c12240b0c44d1ae7ea2d15"

.gds/compiled-policy.json

@@ -2,8 +2,8 @@
   "schema_version": 1,
   "compiled_policy": {
     "repository_id": "repo_01M0EYTYCS3DNXDHZ0CVKXWWW1",
-    "bundle_version": "0.8.0",
-    "digest": "sha256:2ce3acbe4f66efb8486d15781d9866a2120b509cbb78826358290361c8b898ab"
+    "bundle_version": "0.9.4",
+    "digest": "sha256:7a0654a2e86b4e493dc1067ff7ccabb740dbc20a11cf55e77d6bc4411935466c"
   },
   "sources": [
     {

.gds/repository.yaml

@@ -59,7 +59,7 @@ git:
 
 ci:
   profile: "go"
-  go_version: "1.26.6"
+  go_version: "1.27.1"
   build_command: "go build -trimpath ./..."
   test_command: "go test ./..."
   timeout_minutes: 40

Makefile

@@ -1,4 +1,5 @@
 .DEFAULT_GOAL := verify
+GOFMT = $(shell go env GOROOT)/bin/gofmt
 
 .PHONY: build build-controller controller-release build-garm-derivative fmt-check garm-derivative-script test test-controller-release-manifest test-race vet verify
 
@@ -52,8 +53,8 @@ build-garm-derivative:
 	scripts/build-garm-nddev.sh
 
 fmt-check:
-	@test -z "$$(gofmt -l cmd internal third_party)" || { \
-		gofmt -d cmd internal third_party; \
+	@test -z "$$("$(GOFMT)" -l cmd internal third_party)" || { \
+		"$(GOFMT)" -d cmd internal third_party; \
 		exit 1; \
 	}
 

cmd/garm-provider-incus-nddev/main_test.go

@@ -3,6 +3,7 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"os"
 	"strings"
 	"testing"
 )
@@ -79,7 +80,23 @@ func TestVersionCommandReportsBuildAndSDKProvenance(t *testing.T) {
 	if err := json.Unmarshal(stdout.Bytes(), &output); err != nil {
 		t.Fatalf("decode version output: %v", err)
 	}
-	if output["version"] != version || output["commit"] != commit || output["incus_sdk_version"] != "v7.3.0" {
+	// Compare the reported SDK with the independently declared module pin.
+	// A second version literal would let dependency upgrades leave stale provenance.
+	module, err := os.ReadFile("../../go.mod")
+	if err != nil {
+		t.Fatal(err)
+	}
+	expectedSDK := ""
+	for _, line := range strings.Split(string(module), "\n") {
+		fields := strings.Fields(line)
+		if len(fields) >= 2 && fields[0] == "github.com/lxc/incus/v7" {
+			expectedSDK = fields[1]
+		}
+	}
+	if expectedSDK == "" {
+		t.Fatal("Incus SDK module pin is missing")
+	}
+	if output["version"] != version || output["commit"] != commit || output["incus_sdk_version"] != expectedSDK {
 		t.Fatalf("unexpected version output: %#v", output)
 	}
 }

config/example-runner-1.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.131
+  provider_version: v0.1.5-nddev.132
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-2.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.131
+  provider_version: v0.1.5-nddev.132
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-3.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.131
+  provider_version: v0.1.5-nddev.132
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-runner-4.yaml

@@ -9,7 +9,7 @@ control_plane:
   manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.131
+  provider_version: v0.1.5-nddev.132
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/example-services.yaml

@@ -27,7 +27,7 @@ control_plane:
   manager_version: v0.2.1-nddev.94
   scheduling_mode: scale-set
   provider: incus
-  provider_version: v0.1.5-nddev.131
+  provider_version: v0.1.5-nddev.132
   provider_interface: v0.1.0
   worker_kind: incus-container
   runner: actions/runner

config/provider-derivative.yaml

@@ -16,7 +16,7 @@ artifact: garm-provider-incus
 # state all move together, because all three derive from here. A provider change
 # that does not bump it ships under the previous version, which is exactly how
 # runner-1 and runner-2 diverged.
-derivative_version: v0.1.5-nddev.131
+derivative_version: v0.1.5-nddev.132
 
 # The external-provider protocol GARM speaks to this binary. It moves on its own
 # schedule -- a provider release does not imply an interface release -- so it is
@@ -33,13 +33,13 @@ runtime:
   # The Incus client library this provider is compiled against. It decides which
   # Incus API the fleet can speak, so it belongs to the release identity rather
   # than to the code that happens to import it.
-  incus_sdk_version: v7.3.0
+  incus_sdk_version: v7.4.0
   queue_intent_schema_version: 6
 
 build:
-  source_commit: e1493e0b960cbeb561faf1df356767b2933e71ec
-  binary_sha256: 03e7d481fe8969ae55a5a31ea6bcb115e27bc3864d6b64212e0a8b45377732b9
-  go_version: go1.26.7
+  source_commit: 7a82d685b1b658f6aa66a8967ae91ed2b92df71b
+  binary_sha256: 5769660e5d6f2f2efc0925fe55ed5987110f46e60a8a9e58aac061c12189961c
+  go_version: go1.27.1
   cgo_enabled: false
   target_os: linux
   target_arch: amd64

config/provider-rollout-contract.json

@@ -1,14 +1,13 @@
 {
-  "schema_version": 1,
+  "schema_version": 2,
   "ordered_phases": [
-    "platform-policies",
-    "provider-binary-and-config",
-    "manager-restart",
+    "source-and-config-verification",
+    "services-identity-swap",
     "observer-restart",
+    "member-identity-swap",
     "bounded-convergence"
   ],
   "restart_units": [
-    "garm.service",
     "gha-fleet-observer.service"
   ],
   "convergence": {
@@ -18,5 +17,10 @@
     "visible_inventory_must_match_provider": true,
     "queue_identity_must_be_preserved": true,
     "natural_job_required": true
-  }
+  },
+  "conditional_restart_units": [
+    "gha-cache-broker.service",
+    "gha-pressure-observer.service"
+  ],
+  "preserve_manager_for_provider_only_update": true
 }

docs/provider-dependency-updates.md

@@ -22,3 +22,7 @@ Major dependency updates remain separate because grouping must not hide a
 review-significant compatibility change. GARM derivative inputs are rebuilt by
 CI too, but root Go module changes do not change its vendored upstream source;
 its existing reproducible digest must remain identical.
+
+The reported Incus SDK version must agree with the actual module pin; the CLI
+provenance test reads `go.mod` rather than repeating a second expected literal.
+Formatter and nested verification commands use the selected Go toolchain.

docs/provider-rollouts.md

@@ -5,9 +5,12 @@ contract for an Incus provider identity change. Provider binaries and the fleet
 observer both resolve provider identity at process startup, so updating files
 and restarting only GARM leaves inventory observation fail-closed and stale.
 
-Apply the declared phases in order. Deploy and read back all platform policies
-before activating the provider binary and config. Restart the manager, verify
-queue identity preservation, then restart the observer. Acceptance requires a
+Apply contract v2's declared phases in order: verify source/config, swap the
+services identity, restart the observer, swap member identities, and verify
+bounded convergence. A provider-only update preserves the manager process;
+its PID is checked before and after the wave. Changed broker and pressure
+observer binaries restart conditionally. A manager binary/config change is a
+separate rollout. Acceptance requires a
 fresh healthy sample, zero collection errors, inventory parity with the
 provider, and a successful natural job bound to the new provider identity.
 
@@ -23,7 +26,7 @@ learned by omission on a live wave:
    (mode 0640 root:garm is part of the contract);
 3. `previous_provider_identities` — one release of rolling compatibility, the
    only thing letting in-flight work from the outgoing release finish;
-4. the `platform.yaml` provider pin on all five hosts — the observer and
+4. the `platform.yaml` provider pin on every selected services/compute host — the observer and
    controller resolve identity at startup, and a missed pin reads as
    `platform_unhealthy`;
 5. **stale-stamped warm instances**. Warm must be exact-current. Since
@@ -51,5 +54,15 @@ binary reproducibly from the manifest’s own `source_commit` and refuses a
 sha mismatch, deploys services then members in the pin order above, and
 verifies live identity, observer health, zero failed units and warm
 convergence onto the new release. Edit and review the estate toml and the
-five platform pins first — the script deploys reviewed state, it does not
+selected platform pins first — the script deploys reviewed state, it does not
 invent it.
+
+## Incomplete inventory during a lease transition
+
+The Incus list and provider journal are separate observations. A completed
+transition can leave an old metadata-free list entry after its lease disappears.
+The provider repeats the complete inventory observation once for this exact
+condition, retaining replacement workers in the same accounting pass. Active
+leases still account for pending transitions. Persistent ambiguity, unknown
+ownership, image drift and isolation violations still fail closed; the refresh
+itself changes no worker or journal state.

go.mod

@@ -1,6 +1,6 @@
 module github.com/NDDev-OpenNetwork/github-actions
 
-go 1.26.7
+go 1.27.1
 
 require (
 	github.com/BurntSushi/toml v1.6.0
@@ -21,7 +21,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0
 	go.opentelemetry.io/otel/sdk v1.46.0
 	go.opentelemetry.io/otel/trace v1.46.0
-	golang.org/x/crypto v0.56.0
+	golang.org/x/crypto v0.57.0
 	golang.org/x/sys v0.48.0
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -91,8 +91,8 @@ require (
 	go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect
 	golang.org/x/net v0.58.0 // indirect
 	golang.org/x/oauth2 v0.36.0 // indirect
-	golang.org/x/term v0.45.0 // indirect
-	golang.org/x/text v0.41.0 // indirect
+	golang.org/x/term v0.46.0 // indirect
+	golang.org/x/text v0.42.0 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect
 	google.golang.org/grpc v1.83.2 // indirect

go.sum

@@ -257,8 +257,8 @@ go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
 go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y=
-golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I=
+golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M=
+golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
@@ -274,12 +274,12 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
 golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
-golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
-golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
+golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE=
+golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8=
-golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M=
+golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI=
+golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=

internal/admission/admission_test.go

@@ -146,21 +146,21 @@ func TestEvaluateUsesOneLedgerAcrossCapabilityNames(t *testing.T) {
 
 func validInputs() (HostSnapshot, ReservePolicy, Request) {
 	return HostSnapshot{
-			Healthy:            true,
-			TotalCPUUnits:      32,
-			TotalMemoryMiB:     128 * 1024,
-			AvailableMemoryMiB: 128 * 1024,
-			FreeDiskPercent:    50,
-		}, ReservePolicy{
-			MinimumCPUUnits:        4,
-			MinimumMemoryMiB:       16 * 1024,
-			MinimumPercent:         10,
-			MinimumFreeDiskPercent: 20,
-		}, Request{
-			PoolName:  "nddev-linux-standard",
-			VCPU:      4,
-			MemoryMiB: 12 * 1024,
-		}
+		Healthy:            true,
+		TotalCPUUnits:      32,
+		TotalMemoryMiB:     128 * 1024,
+		AvailableMemoryMiB: 128 * 1024,
+		FreeDiskPercent:    50,
+	}, ReservePolicy{
+		MinimumCPUUnits:        4,
+		MinimumMemoryMiB:       16 * 1024,
+		MinimumPercent:         10,
+		MinimumFreeDiskPercent: 20,
+	}, Request{
+		PoolName:  "nddev-linux-standard",
+		VCPU:      4,
+		MemoryMiB: 12 * 1024,
+	}
 }
 
 func TestEvaluatePreservesLiveAvailableMemoryReserve(t *testing.T) {

internal/garmproviderincus/provider/admission.go

@@ -2,6 +2,7 @@ package provider
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
 	"slices"
@@ -331,7 +332,29 @@ func (n *nddevAdmission) diagnosticsBlocked() (bool, error) {
 	return workerdiagnostics.AtDurableWALHighWatermark(stats, n.diagnosticsMaxBytes), nil
 }
 
+var errIncompleteInventory = errors.New("incomplete instance metadata")
+
 func (n *nddevAdmission) observedAllocations(ctx context.Context, cli InstanceServerInterface) ([]provideradmission.Allocation, error) {
+	var allocations []provideradmission.Allocation
+	var err error
+	for attempt := 0; attempt < 2; attempt++ {
+		if err := ctx.Err(); err != nil {
+			return nil, err
+		}
+		allocations, err = n.observeAllocationsOnce(ctx, cli)
+		if !errors.Is(err, errIncompleteInventory) {
+			return allocations, err
+		}
+		// Incus inventory and the lease journal are separate observations.
+		// A completed transition can leave an old metadata-free list entry
+		// after its lease is removed. Re-read the whole inventory once, so
+		// disappeared names and any replacements are accounted together.
+		// Persistent ambiguity still fails closed; no lease or worker changes.
+	}
+	return nil, err
+}
+
+func (n *nddevAdmission) observeAllocationsOnce(ctx context.Context, cli InstanceServerInterface) ([]provideradmission.Allocation, error) {
 	instances, err := cli.GetInstances(api.InstanceTypeAny)
 	if err != nil {
 		return nil, fmt.Errorf("observe Incus allocations: %w", err)
@@ -379,8 +402,8 @@ func (n *nddevAdmission) observedAllocations(ctx context.Context, cli InstanceSe
 			lease, owned := state.Leases[instance.Name]
 			if !owned || (lease.State != providerjournal.StateAdmitted && lease.State != providerjournal.StateCreated && lease.State != providerjournal.StateDeleting) {
 				return nil, fmt.Errorf(
-					"incomplete instance metadata: instance %q has no flavor and no active provider lease",
-					instance.Name,
+					"%w: instance %q has no flavor and no active provider lease",
+					errIncompleteInventory, instance.Name,
 				)
 			}
 			allocations = append(allocations, provideradmission.Allocation{