diff --git a/.gds/bundle.lock.yaml b/.gds/bundle.lock.yaml index 04736116..03e5065e 100644 --- a/.gds/bundle.lock.yaml +++ b/.gds/bundle.lock.yaml @@ -2,16 +2,16 @@ schema_version: 1 bundle: - version: "0.8.0" - release_sequence: 55 + version: "0.9.4" + release_sequence: 61 channel: "stable" - source_tree_digest: "sha256:059be9ccd7b94dbb2ed1d00de45ea0f33de430325aa15f0646c04eb35fd10249" - digest: "sha256:e582619f2c9bf192b57b761418d52afe520630a074cbdae3ae9199223c33c93e" - attestation_identity_digest: "sha256:e1b787c63de916f750e2cbcbf664967669024e1062964515f612b2c826c9e10d" + source_tree_digest: "sha256:7539972f647cb41bde607b2df27ba7edc6f38944cd03bd3998e0399feb81fc38" + digest: "sha256:d1952b8e599e5b71b7a4e37614817b6ec30081003f1380a906b6a62defe2a332" + attestation_identity_digest: "sha256:bae54be9c903b0ec1d356ce3ae4b54b4226f0ae978c3c8d544fccde2d17bd771" projection: - input_digest: "sha256:16d00b8c8cfe471cc00e14ab983f28cbed63448fc13f2f7f35977d827a172f11" - output_digest: "sha256:6df8425d9b3ebe48a39fad7d21493444364c697256721cd692ea35e1658abf0e" + input_digest: "sha256:ac8c5e5d8fa76f0114698ebc2a8653a05f06bdf24ca3eea17c87f4e701657099" + output_digest: "sha256:766c85527ee2ce108bd40901d90751dd048d201d3dee72721aef872e0fd78ac6" files: - path: ".gds/compiled-policy.json" - digest: "sha256:6f72b631e71dfc641101dad2b84e9a4bd6c42281f9c87e168d883aa85d11c20d" + digest: "sha256:71e0f6cb2a4e0e3e34968e900a5f2969b372ceaab6c12240b0c44d1ae7ea2d15" diff --git a/.gds/compiled-policy.json b/.gds/compiled-policy.json index 3d4cda1f..d5458372 100644 --- a/.gds/compiled-policy.json +++ b/.gds/compiled-policy.json @@ -2,8 +2,8 @@ "schema_version": 1, "compiled_policy": { "repository_id": "repo_01M0EYTYCS3DNXDHZ0CVKXWWW1", - "bundle_version": "0.8.0", - "digest": "sha256:2ce3acbe4f66efb8486d15781d9866a2120b509cbb78826358290361c8b898ab" + "bundle_version": "0.9.4", + "digest": "sha256:7a0654a2e86b4e493dc1067ff7ccabb740dbc20a11cf55e77d6bc4411935466c" }, "sources": [ { diff --git a/.gds/repository.yaml b/.gds/repository.yaml index 05551f79..fa04ddb2 100644 --- a/.gds/repository.yaml +++ b/.gds/repository.yaml @@ -59,7 +59,7 @@ git: ci: profile: "go" - go_version: "1.26.6" + go_version: "1.27.1" build_command: "go build -trimpath ./..." test_command: "go test ./..." timeout_minutes: 40 diff --git a/Makefile b/Makefile index 69156267..09715d27 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,5 @@ .DEFAULT_GOAL := verify +GOFMT = $(shell go env GOROOT)/bin/gofmt .PHONY: build build-controller controller-release build-garm-derivative fmt-check garm-derivative-script test test-controller-release-manifest test-race vet verify @@ -52,8 +53,8 @@ build-garm-derivative: scripts/build-garm-nddev.sh fmt-check: - @test -z "$$(gofmt -l cmd internal third_party)" || { \ - gofmt -d cmd internal third_party; \ + @test -z "$$("$(GOFMT)" -l cmd internal third_party)" || { \ + "$(GOFMT)" -d cmd internal third_party; \ exit 1; \ } diff --git a/cmd/garm-provider-incus-nddev/main_test.go b/cmd/garm-provider-incus-nddev/main_test.go index 0dc27642..19ec27f9 100644 --- a/cmd/garm-provider-incus-nddev/main_test.go +++ b/cmd/garm-provider-incus-nddev/main_test.go @@ -3,6 +3,7 @@ package main import ( "bytes" "encoding/json" + "os" "strings" "testing" ) @@ -79,7 +80,23 @@ func TestVersionCommandReportsBuildAndSDKProvenance(t *testing.T) { if err := json.Unmarshal(stdout.Bytes(), &output); err != nil { t.Fatalf("decode version output: %v", err) } - if output["version"] != version || output["commit"] != commit || output["incus_sdk_version"] != "v7.3.0" { + // Compare the reported SDK with the independently declared module pin. + // A second version literal would let dependency upgrades leave stale provenance. + module, err := os.ReadFile("../../go.mod") + if err != nil { + t.Fatal(err) + } + expectedSDK := "" + for _, line := range strings.Split(string(module), "\n") { + fields := strings.Fields(line) + if len(fields) >= 2 && fields[0] == "github.com/lxc/incus/v7" { + expectedSDK = fields[1] + } + } + if expectedSDK == "" { + t.Fatal("Incus SDK module pin is missing") + } + if output["version"] != version || output["commit"] != commit || output["incus_sdk_version"] != expectedSDK { t.Fatalf("unexpected version output: %#v", output) } } diff --git a/config/example-runner-1.yaml b/config/example-runner-1.yaml index 3c73d994..b1d267f8 100644 --- a/config/example-runner-1.yaml +++ b/config/example-runner-1.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.131 + provider_version: v0.1.5-nddev.132 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-2.yaml b/config/example-runner-2.yaml index 37ed5070..408d73e5 100644 --- a/config/example-runner-2.yaml +++ b/config/example-runner-2.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.131 + provider_version: v0.1.5-nddev.132 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-3.yaml b/config/example-runner-3.yaml index 56e5f1d4..a2101ca4 100644 --- a/config/example-runner-3.yaml +++ b/config/example-runner-3.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.131 + provider_version: v0.1.5-nddev.132 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-runner-4.yaml b/config/example-runner-4.yaml index e750ecb5..a715914e 100644 --- a/config/example-runner-4.yaml +++ b/config/example-runner-4.yaml @@ -9,7 +9,7 @@ control_plane: manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.131 + provider_version: v0.1.5-nddev.132 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/example-services.yaml b/config/example-services.yaml index efe4b27c..033bbe9a 100644 --- a/config/example-services.yaml +++ b/config/example-services.yaml @@ -27,7 +27,7 @@ control_plane: manager_version: v0.2.1-nddev.94 scheduling_mode: scale-set provider: incus - provider_version: v0.1.5-nddev.131 + provider_version: v0.1.5-nddev.132 provider_interface: v0.1.0 worker_kind: incus-container runner: actions/runner diff --git a/config/provider-derivative.yaml b/config/provider-derivative.yaml index 9ad69668..14057917 100644 --- a/config/provider-derivative.yaml +++ b/config/provider-derivative.yaml @@ -16,7 +16,7 @@ artifact: garm-provider-incus # state all move together, because all three derive from here. A provider change # that does not bump it ships under the previous version, which is exactly how # runner-1 and runner-2 diverged. -derivative_version: v0.1.5-nddev.131 +derivative_version: v0.1.5-nddev.132 # The external-provider protocol GARM speaks to this binary. It moves on its own # schedule -- a provider release does not imply an interface release -- so it is @@ -33,13 +33,13 @@ runtime: # The Incus client library this provider is compiled against. It decides which # Incus API the fleet can speak, so it belongs to the release identity rather # than to the code that happens to import it. - incus_sdk_version: v7.3.0 + incus_sdk_version: v7.4.0 queue_intent_schema_version: 6 build: - source_commit: e1493e0b960cbeb561faf1df356767b2933e71ec - binary_sha256: 03e7d481fe8969ae55a5a31ea6bcb115e27bc3864d6b64212e0a8b45377732b9 - go_version: go1.26.7 + source_commit: 7a82d685b1b658f6aa66a8967ae91ed2b92df71b + binary_sha256: 5769660e5d6f2f2efc0925fe55ed5987110f46e60a8a9e58aac061c12189961c + go_version: go1.27.1 cgo_enabled: false target_os: linux target_arch: amd64 diff --git a/config/provider-rollout-contract.json b/config/provider-rollout-contract.json index 007e0022..b911db23 100644 --- a/config/provider-rollout-contract.json +++ b/config/provider-rollout-contract.json @@ -1,14 +1,13 @@ { - "schema_version": 1, + "schema_version": 2, "ordered_phases": [ - "platform-policies", - "provider-binary-and-config", - "manager-restart", + "source-and-config-verification", + "services-identity-swap", "observer-restart", + "member-identity-swap", "bounded-convergence" ], "restart_units": [ - "garm.service", "gha-fleet-observer.service" ], "convergence": { @@ -18,5 +17,10 @@ "visible_inventory_must_match_provider": true, "queue_identity_must_be_preserved": true, "natural_job_required": true - } + }, + "conditional_restart_units": [ + "gha-cache-broker.service", + "gha-pressure-observer.service" + ], + "preserve_manager_for_provider_only_update": true } diff --git a/docs/provider-dependency-updates.md b/docs/provider-dependency-updates.md index 11b08617..8b639b95 100644 --- a/docs/provider-dependency-updates.md +++ b/docs/provider-dependency-updates.md @@ -22,3 +22,7 @@ Major dependency updates remain separate because grouping must not hide a review-significant compatibility change. GARM derivative inputs are rebuilt by CI too, but root Go module changes do not change its vendored upstream source; its existing reproducible digest must remain identical. + +The reported Incus SDK version must agree with the actual module pin; the CLI +provenance test reads `go.mod` rather than repeating a second expected literal. +Formatter and nested verification commands use the selected Go toolchain. diff --git a/docs/provider-rollouts.md b/docs/provider-rollouts.md index 744a9a13..48045d6d 100644 --- a/docs/provider-rollouts.md +++ b/docs/provider-rollouts.md @@ -5,9 +5,12 @@ contract for an Incus provider identity change. Provider binaries and the fleet observer both resolve provider identity at process startup, so updating files and restarting only GARM leaves inventory observation fail-closed and stale. -Apply the declared phases in order. Deploy and read back all platform policies -before activating the provider binary and config. Restart the manager, verify -queue identity preservation, then restart the observer. Acceptance requires a +Apply contract v2's declared phases in order: verify source/config, swap the +services identity, restart the observer, swap member identities, and verify +bounded convergence. A provider-only update preserves the manager process; +its PID is checked before and after the wave. Changed broker and pressure +observer binaries restart conditionally. A manager binary/config change is a +separate rollout. Acceptance requires a fresh healthy sample, zero collection errors, inventory parity with the provider, and a successful natural job bound to the new provider identity. @@ -23,7 +26,7 @@ learned by omission on a live wave: (mode 0640 root:garm is part of the contract); 3. `previous_provider_identities` — one release of rolling compatibility, the only thing letting in-flight work from the outgoing release finish; -4. the `platform.yaml` provider pin on all five hosts — the observer and +4. the `platform.yaml` provider pin on every selected services/compute host — the observer and controller resolve identity at startup, and a missed pin reads as `platform_unhealthy`; 5. **stale-stamped warm instances**. Warm must be exact-current. Since @@ -51,5 +54,15 @@ binary reproducibly from the manifest’s own `source_commit` and refuses a sha mismatch, deploys services then members in the pin order above, and verifies live identity, observer health, zero failed units and warm convergence onto the new release. Edit and review the estate toml and the -five platform pins first — the script deploys reviewed state, it does not +selected platform pins first — the script deploys reviewed state, it does not invent it. + +## Incomplete inventory during a lease transition + +The Incus list and provider journal are separate observations. A completed +transition can leave an old metadata-free list entry after its lease disappears. +The provider repeats the complete inventory observation once for this exact +condition, retaining replacement workers in the same accounting pass. Active +leases still account for pending transitions. Persistent ambiguity, unknown +ownership, image drift and isolation violations still fail closed; the refresh +itself changes no worker or journal state. diff --git a/go.mod b/go.mod index 8f4c137b..2136de01 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/NDDev-OpenNetwork/github-actions -go 1.26.7 +go 1.27.1 require ( github.com/BurntSushi/toml v1.6.0 @@ -21,7 +21,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.46.0 go.opentelemetry.io/otel/sdk v1.46.0 go.opentelemetry.io/otel/trace v1.46.0 - golang.org/x/crypto v0.56.0 + golang.org/x/crypto v0.57.0 golang.org/x/sys v0.48.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -91,8 +91,8 @@ require ( go.yaml.in/yaml/v4 v4.0.0-rc.6 // indirect golang.org/x/net v0.58.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect - golang.org/x/term v0.45.0 // indirect - golang.org/x/text v0.41.0 // indirect + golang.org/x/term v0.46.0 // indirect + golang.org/x/text v0.42.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260819154853-08b0e4226688 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260825221802-da73d73af1c5 // indirect google.golang.org/grpc v1.83.2 // indirect diff --git a/go.sum b/go.sum index cc2f1dce..b4a2c39f 100644 --- a/go.sum +++ b/go.sum @@ -257,8 +257,8 @@ go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.56.0 h1:GUh5Ii4J5jtcseSMiRqr1jXCNHoxjeV9Fmekc2oLy6Y= -golang.org/x/crypto v0.56.0/go.mod h1:OMW5y6CY9l38uPLmxU6l6pwcXp1obtLo3e6gT7gQR2I= +golang.org/x/crypto v0.57.0 h1:3ZVCjf8Ggz7zneR/EHRVx68Ctf+2pmIMP2UFhh9cC6M= +golang.org/x/crypto v0.57.0/go.mod h1:Fdz0i5U6CoizGwLda9DttjSk6qlZo25zYNtR+ycvuZA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -274,12 +274,12 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo= golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= -golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= -golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/term v0.46.0 h1:3+OXuTbaKDgwk8jTi3aSLHRlmWqHEUDUtxnbFigO4YE= +golang.org/x/term v0.46.0/go.mod h1:+K02xbkittuwc0Am4abfA3Fc+XRGXkvBXNO88NCXPoc= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= -golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= +golang.org/x/text v0.42.0 h1:JbOZXgfeCPU9gacVtYliJqOhD+zhrEqK4LfdpmlUZqI= +golang.org/x/text v0.42.0/go.mod h1:ojzP1Z+2QtioaF8DTtO8K5q7JWVVYwZKenzujK0Zd0E= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= diff --git a/internal/admission/admission_test.go b/internal/admission/admission_test.go index 25274c43..3029d4ff 100644 --- a/internal/admission/admission_test.go +++ b/internal/admission/admission_test.go @@ -146,21 +146,21 @@ func TestEvaluateUsesOneLedgerAcrossCapabilityNames(t *testing.T) { func validInputs() (HostSnapshot, ReservePolicy, Request) { return HostSnapshot{ - Healthy: true, - TotalCPUUnits: 32, - TotalMemoryMiB: 128 * 1024, - AvailableMemoryMiB: 128 * 1024, - FreeDiskPercent: 50, - }, ReservePolicy{ - MinimumCPUUnits: 4, - MinimumMemoryMiB: 16 * 1024, - MinimumPercent: 10, - MinimumFreeDiskPercent: 20, - }, Request{ - PoolName: "nddev-linux-standard", - VCPU: 4, - MemoryMiB: 12 * 1024, - } + Healthy: true, + TotalCPUUnits: 32, + TotalMemoryMiB: 128 * 1024, + AvailableMemoryMiB: 128 * 1024, + FreeDiskPercent: 50, + }, ReservePolicy{ + MinimumCPUUnits: 4, + MinimumMemoryMiB: 16 * 1024, + MinimumPercent: 10, + MinimumFreeDiskPercent: 20, + }, Request{ + PoolName: "nddev-linux-standard", + VCPU: 4, + MemoryMiB: 12 * 1024, + } } func TestEvaluatePreservesLiveAvailableMemoryReserve(t *testing.T) { diff --git a/internal/garmproviderincus/provider/admission.go b/internal/garmproviderincus/provider/admission.go index 673d4106..17331cdf 100644 --- a/internal/garmproviderincus/provider/admission.go +++ b/internal/garmproviderincus/provider/admission.go @@ -2,6 +2,7 @@ package provider import ( "context" + "errors" "fmt" "os" "slices" @@ -331,7 +332,29 @@ func (n *nddevAdmission) diagnosticsBlocked() (bool, error) { return workerdiagnostics.AtDurableWALHighWatermark(stats, n.diagnosticsMaxBytes), nil } +var errIncompleteInventory = errors.New("incomplete instance metadata") + func (n *nddevAdmission) observedAllocations(ctx context.Context, cli InstanceServerInterface) ([]provideradmission.Allocation, error) { + var allocations []provideradmission.Allocation + var err error + for attempt := 0; attempt < 2; attempt++ { + if err := ctx.Err(); err != nil { + return nil, err + } + allocations, err = n.observeAllocationsOnce(ctx, cli) + if !errors.Is(err, errIncompleteInventory) { + return allocations, err + } + // Incus inventory and the lease journal are separate observations. + // A completed transition can leave an old metadata-free list entry + // after its lease is removed. Re-read the whole inventory once, so + // disappeared names and any replacements are accounted together. + // Persistent ambiguity still fails closed; no lease or worker changes. + } + return nil, err +} + +func (n *nddevAdmission) observeAllocationsOnce(ctx context.Context, cli InstanceServerInterface) ([]provideradmission.Allocation, error) { instances, err := cli.GetInstances(api.InstanceTypeAny) if err != nil { return nil, fmt.Errorf("observe Incus allocations: %w", err) @@ -379,8 +402,8 @@ func (n *nddevAdmission) observedAllocations(ctx context.Context, cli InstanceSe lease, owned := state.Leases[instance.Name] if !owned || (lease.State != providerjournal.StateAdmitted && lease.State != providerjournal.StateCreated && lease.State != providerjournal.StateDeleting) { return nil, fmt.Errorf( - "incomplete instance metadata: instance %q has no flavor and no active provider lease", - instance.Name, + "%w: instance %q has no flavor and no active provider lease", + errIncompleteInventory, instance.Name, ) } allocations = append(allocations, provideradmission.Allocation{ diff --git a/internal/garmproviderincus/provider/admission_test.go b/internal/garmproviderincus/provider/admission_test.go index e84b31ed..d0ae317c 100644 --- a/internal/garmproviderincus/provider/admission_test.go +++ b/internal/garmproviderincus/provider/admission_test.go @@ -158,13 +158,62 @@ func TestObservedAllocationsClassifyUnjournaledIncompleteMetadata(t *testing.T) require.NoError(t, err) instance := ownedInstance("runner-incomplete") delete(instance.ExpandedConfig, flavorKey) - cli.On("GetInstances", api.InstanceTypeAny).Return([]api.InstanceFull{*instance}, nil).Once() + cli.On("GetInstances", api.InstanceTypeAny).Return([]api.InstanceFull{*instance}, nil).Twice() _, err = admission.observedAllocations(context.Background(), cli) require.ErrorContains(t, err, "incomplete instance metadata") require.ErrorContains(t, err, "no flavor and no active provider lease") } +func TestObservedAllocationsRefreshUnjournaledIncompleteSnapshot(t *testing.T) { + t.Parallel() + foreign := ownedInstance("runner-raced") + foreign.ExpandedConfig[controllerIDKeyName] = "foreign-controller" + wrongType := ownedInstance("runner-raced") + wrongType.Type = string(api.InstanceTypeContainer) + for _, testCase := range []struct { + name string + instances []api.InstanceFull + err error + wantNames []string + wantError string + }{ + {name: "deleted after listing"}, + {name: "metadata completed after listing", instances: []api.InstanceFull{*ownedInstance("runner-raced")}, wantNames: []string{"runner-raced"}}, + {name: "replacement is accounted", instances: []api.InstanceFull{*ownedInstance("replacement-runner")}, wantNames: []string{"replacement-runner"}}, + {name: "stopped after listing", instances: []api.InstanceFull{{Instance: api.Instance{Name: "runner-raced", Status: "Stopped"}}}}, + {name: "refresh unavailable", err: os.ErrPermission, wantError: "observe Incus allocations"}, + {name: "foreign refreshed owner", instances: []api.InstanceFull{*foreign}, wantError: controllerIDKeyName}, + {name: "wrong refreshed isolation type", instances: []api.InstanceFull{*wrongType}, wantError: "has type"}, + {name: "still incomplete", instances: []api.InstanceFull{{Instance: api.Instance{Name: "runner-raced"}}}, wantError: "no flavor and no active provider lease"}, + } { + t.Run(testCase.name, func(t *testing.T) { + admission := testNDDevAdmission() + directory := t.TempDir() + admission.controller.Store = providerjournal.Store{ + Path: filepath.Join(directory, "journal.json"), LockPath: filepath.Join(directory, "journal.lock"), + } + _, err := admission.controller.Store.Update(context.Background(), func(*providerjournal.Journal) error { return nil }) + require.NoError(t, err) + cli := new(MockIncusServer) + first := cli.On("GetInstances", api.InstanceTypeAny).Return([]api.InstanceFull{{Instance: api.Instance{Name: "runner-raced"}}}, nil).Once() + cli.On("GetInstances", api.InstanceTypeAny).Return(testCase.instances, testCase.err).Once().NotBefore(first) + allocations, err := admission.observedAllocations(context.Background(), cli) + if testCase.wantError != "" { + require.ErrorContains(t, err, testCase.wantError) + } else { + require.NoError(t, err) + require.Len(t, allocations, len(testCase.wantNames)) + for i, name := range testCase.wantNames { + require.Equal(t, name, allocations[i].InstanceName) + require.Equal(t, 10240, allocations[i].MemoryMiB) + } + } + cli.AssertExpectations(t) + }) + } +} + func TestReconcileRetainsDeletingLeaseUntilClusterTombstoneDisappears(t *testing.T) { t.Parallel() admission := testNDDevAdmission() diff --git a/internal/garmproviderincus/provider/incus.go b/internal/garmproviderincus/provider/incus.go index 8b9a5671..c044ff1f 100644 --- a/internal/garmproviderincus/provider/incus.go +++ b/internal/garmproviderincus/provider/incus.go @@ -63,7 +63,7 @@ var _ execution.ExternalProvider = &Incus{} var Version = "v0.0.0-unknown" var Commit = "unknown" -const IncusSDKVersion = "v7.3.0" +const IncusSDKVersion = "v7.4.0" const ( expectedCallbackURL = "https://198.51.100.1:9443/api/v1/callbacks" diff --git a/internal/incusplacement/starlarkexec/go.mod b/internal/incusplacement/starlarkexec/go.mod index 36e9d213..e088e2b8 100644 --- a/internal/incusplacement/starlarkexec/go.mod +++ b/internal/incusplacement/starlarkexec/go.mod @@ -2,7 +2,7 @@ // go.mod/go.sum stay identical to the pinned provider derivative. module github.com/NDDev-OpenNetwork/github-actions/internal/incusplacement/starlarkexec -go 1.26.7 +go 1.27.1 require ( github.com/NDDev-OpenNetwork/github-actions v0.0.0 diff --git a/internal/incusplacement/starlarkexec_hook_test.go b/internal/incusplacement/starlarkexec_hook_test.go index 57fd2b06..6f7fcf3f 100644 --- a/internal/incusplacement/starlarkexec_hook_test.go +++ b/internal/incusplacement/starlarkexec_hook_test.go @@ -21,7 +21,7 @@ func TestStarlarkExecutionHarness(t *testing.T) { if raceEnabled { args = []string{"test", "-race", "-count=1", "."} } - cmd := exec.Command("go", args...) + cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), args...) cmd.Dir = dir cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local") output, err := cmd.CombinedOutput() @@ -36,7 +36,7 @@ func TestStarlarkExecutionHarnessVet(t *testing.T) { t.Fatal("runtime.Caller failed") } dir := filepath.Join(filepath.Dir(file), "starlarkexec") - cmd := exec.Command("go", "vet", ".") + cmd := exec.Command(filepath.Join(runtime.GOROOT(), "bin", "go"), "vet", ".") cmd.Dir = dir cmd.Env = append(os.Environ(), "GOTOOLCHAIN=local") output, err := cmd.CombinedOutput() diff --git a/internal/providerrollout/contract.go b/internal/providerrollout/contract.go index e4753316..e7579b44 100644 --- a/internal/providerrollout/contract.go +++ b/internal/providerrollout/contract.go @@ -8,10 +8,12 @@ import ( ) type Contract struct { - SchemaVersion int `json:"schema_version"` - OrderedPhases []string `json:"ordered_phases"` - RestartUnits []string `json:"restart_units"` - Convergence struct { + SchemaVersion int `json:"schema_version"` + OrderedPhases []string `json:"ordered_phases"` + RestartUnits []string `json:"restart_units"` + ConditionalRestartUnits []string `json:"conditional_restart_units"` + PreserveManager bool `json:"preserve_manager_for_provider_only_update"` + Convergence struct { HealthRequired bool `json:"health_required"` FreshSampleRequired bool `json:"fresh_sample_required"` CollectionErrorsMustEqual int `json:"collection_errors_must_equal"` @@ -38,20 +40,21 @@ func Load(path string) (Contract, error) { func (contract Contract) Validate() error { wantPhases := []string{ - "platform-policies", - "provider-binary-and-config", - "manager-restart", + "source-and-config-verification", + "services-identity-swap", "observer-restart", + "member-identity-swap", "bounded-convergence", } - if contract.SchemaVersion != 1 { + if contract.SchemaVersion != 2 { return fmt.Errorf("unsupported rollout contract schema %d", contract.SchemaVersion) } if !slices.Equal(contract.OrderedPhases, wantPhases) { return fmt.Errorf("provider rollout phases must be %v", wantPhases) } - if !slices.Equal(contract.RestartUnits, []string{"garm.service", "gha-fleet-observer.service"}) { - return fmt.Errorf("provider rollout must restart manager then observer") + if !contract.PreserveManager || !slices.Equal(contract.RestartUnits, []string{"gha-fleet-observer.service"}) || + !slices.Equal(contract.ConditionalRestartUnits, []string{"gha-cache-broker.service", "gha-pressure-observer.service"}) { + return fmt.Errorf("provider-only rollout must preserve the manager, restart the observer, and restart changed broker/pressure binaries") } convergence := contract.Convergence if !convergence.HealthRequired || !convergence.FreshSampleRequired || diff --git a/internal/providerrollout/contract_test.go b/internal/providerrollout/contract_test.go index c3ea0feb..f02d4f2a 100644 --- a/internal/providerrollout/contract_test.go +++ b/internal/providerrollout/contract_test.go @@ -11,7 +11,26 @@ func TestProviderRolloutContract(t *testing.T) { if err != nil { t.Fatal(err) } - if contract.OrderedPhases[3] != "observer-restart" { - t.Fatal("observer restart is not ordered after manager restart") + if contract.OrderedPhases[2] != "observer-restart" { + t.Fatal("observer restart is not ordered after the services identity swap") + } +} + +func TestProviderOnlyRolloutCannotRestartManagerOrOmitConvergence(t *testing.T) { + contract, err := Load(filepath.Join("..", "..", "config", "provider-rollout-contract.json")) + if err != nil { + t.Fatal(err) + } + for _, mutate := range []func(*Contract){ + func(c *Contract) { c.PreserveManager = false }, + func(c *Contract) { c.RestartUnits = append(c.RestartUnits, "garm.service") }, + func(c *Contract) { c.Convergence.NaturalJobRequired = false }, + func(c *Contract) { c.OrderedPhases = []string{"bounded-convergence"} }, + } { + changed := contract + mutate(&changed) + if changed.Validate() == nil { + t.Fatal("unsafe or incomplete rollout contract accepted") + } } }