diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec6afca..7af47e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,14 +3,93 @@ name: CI on: workflow_dispatch: +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + jobs: - validate-codex: - runs-on: macos-15 + packages: + runs-on: ubuntu-24.04-arm + timeout-minutes: 15 + outputs: + previous_tag: ${{ steps.candidate.outputs.previous_tag }} + previous_version: ${{ steps.candidate.outputs.previous_version }} + current_version: ${{ steps.candidate.outputs.current_version }} steps: - uses: actions/checkout@v5 with: fetch-depth: 0 + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Create isolated release candidate tag + id: candidate + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + set -euo pipefail + latest_tag= + while IFS= read -r candidate; do + if [[ "$candidate" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + latest_tag="$candidate" + break + fi + done < <(git tag --sort=-v:refname) + previous_tag="$(gh release list \ + --repo "$GITHUB_REPOSITORY" \ + --exclude-drafts \ + --exclude-pre-releases \ + --limit 1 \ + --json tagName \ + --jq '.[0].tagName')" + test -n "$latest_tag" + test -n "$previous_tag" + previous_version="${previous_tag#v}" + latest_version="${latest_tag#v}" + IFS=. read -r major minor patch <<<"$latest_version" + current_version="${major}.${minor}.$((patch + 1))-ci" + git tag "v$current_version" + { + echo "PREVIOUS_TAG=$previous_tag" + echo "PREVIOUS_VERSION=$previous_version" + echo "CURRENT_VERSION=$current_version" + echo "CODEX_STAGE11_PROVENANCE_SHA256=$(go run ./.github/scripts/codex-stage11-provenance.go "$current_version" ./internal/proxy/testdata/codex_stage11_reviewed_manifest.json)" + } >> "$GITHUB_ENV" + { + echo "previous_tag=$previous_tag" + echo "previous_version=$previous_version" + echo "current_version=$current_version" + } >> "$GITHUB_OUTPUT" + + - uses: goreleaser/goreleaser-action@v7 + with: + version: "~> v2" + args: release --clean --skip=publish + env: + GEMINI_ANTIGRAVITY_CLIENT_SECRET: validation + + - uses: actions/upload-artifact@v4 + with: + name: ci-darwin-packages + path: | + dist/*_darwin_*.tar.gz + dist/homebrew/Casks/cq.rb + if-no-files-found: error + retention-days: 1 + compression-level: 0 + + validate-codex: + needs: packages + runs-on: macos-15 + timeout-minutes: 15 + steps: + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 with: go-version-file: go.mod @@ -36,15 +115,10 @@ jobs: grep -F -- "--- PASS: ${test_name} " "$test_log" done - - name: Bind reviewed Codex corpus provenance - run: echo "CODEX_STAGE11_PROVENANCE_SHA256=$(go run ./.github/scripts/codex-stage11-provenance.go 0.0.0 ./internal/proxy/testdata/codex_stage11_reviewed_manifest.json)" >> "$GITHUB_ENV" - - - uses: goreleaser/goreleaser-action@v7 + - uses: actions/download-artifact@v4 with: - version: "~> v2" - args: release --snapshot --clean --skip=publish - env: - GEMINI_ANTIGRAVITY_CLIENT_SECRET: validation + name: ci-darwin-packages + path: dist - name: Validate Homebrew Cask quarantine handling run: | @@ -57,55 +131,24 @@ jobs: .github/scripts/validate-homebrew-cask.sh "$PWD/dist/homebrew/Casks/cq.rb" "$archive" homebrew-lifecycle: + needs: packages runs-on: macos-15 + timeout-minutes: 15 + env: + PREVIOUS_TAG: ${{ needs.packages.outputs.previous_tag }} + PREVIOUS_VERSION: ${{ needs.packages.outputs.previous_version }} + CURRENT_VERSION: ${{ needs.packages.outputs.current_version }} steps: - uses: actions/checkout@v5 - with: - fetch-depth: 0 - uses: actions/setup-go@v6 with: go-version-file: go.mod - - name: Create isolated release candidate tag - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - set -euo pipefail - latest_tag= - while IFS= read -r candidate; do - if [[ "$candidate" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then - latest_tag="$candidate" - break - fi - done < <(git tag --sort=-v:refname) - previous_tag="$(gh release list \ - --repo "$GITHUB_REPOSITORY" \ - --exclude-drafts \ - --exclude-pre-releases \ - --limit 1 \ - --json tagName \ - --jq '.[0].tagName')" - test -n "$latest_tag" - test -n "$previous_tag" - previous_version="${previous_tag#v}" - latest_version="${latest_tag#v}" - IFS=. read -r major minor patch <<<"$latest_version" - current_version="${major}.${minor}.$((patch + 1))-ci" - git tag "v$current_version" - { - echo "PREVIOUS_TAG=$previous_tag" - echo "PREVIOUS_VERSION=$previous_version" - echo "CURRENT_VERSION=$current_version" - echo "CODEX_STAGE11_PROVENANCE_SHA256=$(go run ./.github/scripts/codex-stage11-provenance.go "$current_version" ./internal/proxy/testdata/codex_stage11_reviewed_manifest.json)" - } >> "$GITHUB_ENV" - - - uses: goreleaser/goreleaser-action@v7 + - uses: actions/download-artifact@v4 with: - version: "~> v2" - args: release --clean --skip=publish - env: - GEMINI_ANTIGRAVITY_CLIENT_SECRET: validation + name: ci-darwin-packages + path: dist - name: Validate real Cask install, upgrade, transport, and uninstall env: @@ -133,6 +176,7 @@ jobs: "$CURRENT_VERSION" linux: + timeout-minutes: 15 strategy: fail-fast: false matrix: @@ -159,45 +203,25 @@ jobs: - run: go build ./... - run: go vet ./... - - run: go test -race -count=1 ./internal/installer ./cmd/cq-install - - name: Gate degraded rescue transport - run: | - test_name=TestRuntimeSupervisorDegradedRescueRelaysHTTPAndWebSocketOverTransport - test_log="$RUNNER_TEMP/degraded-rescue-transport.log" - if ! go test -race -count=1 -v ./internal/proxy -run "^${test_name}$" >"$test_log" 2>&1; then - cat "$test_log" >&2 - exit 1 - fi - cat "$test_log" - grep -F -- "--- PASS: ${test_name} " "$test_log" - - name: Gate normal proxy transport - run: | - test_name=TestNormalProxyTransportWebSocketHardLimitMigratesBeforeLeak - test_log="$RUNNER_TEMP/normal-proxy-transport.log" - if ! go test -race -count=1 -v ./internal/proxy -run '^TestNormalProxyTransport(HTTP|WebSocket)' >"$test_log" 2>&1; then - cat "$test_log" >&2 - exit 1 - fi - cat "$test_log" - grep -F -- "--- PASS: ${test_name} " "$test_log" - - name: Gate stock-AppArmor confinement and cleanup + - name: Run full race suite and require native transport gates shell: bash run: | set -euo pipefail - tests='TestLinuxAcceptanceConfinementUsesNamespacesRelaysAndLandlock TestLinuxAcceptanceCancellationReapsDescendants' - test_log="$RUNNER_TEMP/linux-confinement.log" - if ! go test -race -count=1 -v ./internal/proxy -run '^(TestLinuxAcceptanceConfinementUsesNamespacesRelaysAndLandlock|TestLinuxAcceptanceCancellationReapsDescendants)$' >"$test_log" 2>&1; then + test_log="$RUNNER_TEMP/linux-tests.log" + # Includes ./internal/installer and ./cmd/cq-install once per architecture. + if ! go test -race -count=1 ./... -v >"$test_log" 2>&1; then cat "$test_log" >&2 exit 1 fi cat "$test_log" + tests='TestRuntimeSupervisorDegradedRescueRelaysHTTPAndWebSocketOverTransport TestNormalProxyTransportWebSocketHardLimitMigratesBeforeLeak TestLinuxAcceptanceConfinementUsesNamespacesRelaysAndLandlock TestLinuxAcceptanceCancellationReapsDescendants' for test_name in $tests; do grep -F -- "--- PASS: ${test_name} " "$test_log" done - - run: go test -race -count=1 ./... windows: runs-on: windows-latest + timeout-minutes: 15 steps: - uses: actions/checkout@v5 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 345748f..363ccaa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,6 +3,10 @@ name: Release on: workflow_dispatch: +concurrency: + group: release-publication + cancel-in-progress: false + permissions: contents: write statuses: read @@ -10,6 +14,7 @@ permissions: jobs: validate-codex: runs-on: macos-15 + timeout-minutes: 30 steps: - uses: actions/checkout@v5 @@ -38,26 +43,6 @@ jobs: grep -F -- "--- PASS: ${test_name} " "$test_log" done - release: - needs: validate-codex - runs-on: macos-15 - steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - - uses: actions/setup-go@v6 - with: - go-version-file: go.mod - - - name: Require live Codex routing validation - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - sha="$(git rev-parse HEAD)" - gh api "repos/${GITHUB_REPOSITORY}/commits/${sha}/status" | - go run ./internal/tools/releasestatus --sha "${sha}" - - name: Gate degraded rescue transport run: | test_name=TestRuntimeSupervisorDegradedRescueRelaysHTTPAndWebSocketOverTransport @@ -83,6 +68,27 @@ jobs: - name: Verify Codex Stage 11 corpus run: go test -race -count=1 ./internal/proxy -run '^(TestCodexStage11LifecycleCorpusUsesProductionHandlers|TestCodexStage11LifecycleCorpusSmoke|TestCodexStage11ReviewedManifestMatchesCorpusAuthority)$' + draft-build: + needs: validate-codex + runs-on: ubuntu-24.04-arm + timeout-minutes: 20 + steps: + - uses: actions/checkout@v5 + with: + fetch-depth: 0 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - name: Require live Codex routing validation + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + sha="$(git rev-parse HEAD)" + gh api "repos/${GITHUB_REPOSITORY}/commits/${sha}/status" | + go run ./internal/tools/releasestatus --sha "${sha}" + - name: Bind reviewed Codex corpus provenance run: echo "CODEX_STAGE11_PROVENANCE_SHA256=$(go run ./.github/scripts/codex-stage11-provenance.go "${GITHUB_REF_NAME#v}" ./internal/proxy/testdata/codex_stage11_reviewed_manifest.json)" >> "$GITHUB_ENV" @@ -101,6 +107,32 @@ jobs: HOMEBREW_TAP_TOKEN: ${{ secrets.HOMEBREW_TAP_TOKEN }} GEMINI_ANTIGRAVITY_CLIENT_SECRET: ${{ secrets.GEMINI_ANTIGRAVITY_CLIENT_SECRET }} + - uses: actions/upload-artifact@v4 + with: + name: release-homebrew-inputs + path: | + dist/cq_*_darwin_*.tar.gz + dist/homebrew/Casks/cq.rb + if-no-files-found: error + retention-days: 7 + compression-level: 0 + + release: + needs: draft-build + runs-on: macos-15 + timeout-minutes: 30 + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-go@v6 + with: + go-version-file: go.mod + + - uses: actions/download-artifact@v5 + with: + name: release-homebrew-inputs + path: dist + - name: Format draft Homebrew Cask run: | brew style --fix dist/homebrew/Casks/cq.rb @@ -157,10 +189,12 @@ jobs: name: homebrew-cask path: dist/homebrew/Casks/cq.rb if-no-files-found: error + retention-days: 7 windows-packages: needs: release runs-on: windows-latest + timeout-minutes: 30 steps: - uses: actions/checkout@v5 @@ -285,12 +319,15 @@ jobs: name: winget-manifests path: ${{ runner.temp }}/cq-release/winget/manifests if-no-files-found: error + retention-days: 7 - uses: actions/upload-artifact@v4 with: name: windows-msi-validation path: ${{ runner.temp }}/cq-release/cq_*_windows_*.msi if-no-files-found: error + retention-days: 7 + compression-level: 0 windows-acceptance: needs: windows-packages @@ -299,6 +336,7 @@ jobs: matrix: runner: [windows-latest, windows-11-arm] runs-on: ${{ matrix.runner }} + timeout-minutes: 30 steps: - uses: actions/checkout@v5 with: @@ -333,6 +371,7 @@ jobs: linux-install: needs: publish runs-on: ubuntu-24.04-arm + timeout-minutes: 30 steps: - uses: actions/checkout@v5 with: @@ -363,6 +402,7 @@ jobs: publish: needs: [release, windows-packages, windows-acceptance] runs-on: macos-15 + timeout-minutes: 20 steps: - uses: actions/checkout@v5 @@ -454,6 +494,7 @@ jobs: architecture: arm64 winget: false runs-on: ${{ matrix.runner }} + timeout-minutes: 45 steps: - uses: actions/checkout@v5 with: @@ -591,6 +632,7 @@ jobs: winget-publish: needs: [windows-deployed, linux-install] runs-on: ubuntu-latest + timeout-minutes: 15 steps: - uses: actions/download-artifact@v5 with: diff --git a/internal/proxy/codex_frozen_dispatch_plan.go b/internal/proxy/codex_frozen_dispatch_plan.go index bc63811..6a28a85 100644 --- a/internal/proxy/codex_frozen_dispatch_plan.go +++ b/internal/proxy/codex_frozen_dispatch_plan.go @@ -136,6 +136,24 @@ func BuildCodexFrozenDispatchPlan(ctx context.Context, input CodexFrozenDispatch if err != nil { return plan, err } + if len(policy.Choices()) == 0 && policyBoundAccountKey == "" { + // Preserve quota exhaustion when no default route can be attempted. + // Missing credentials or incompatible models are not quota failures. + exhausted := false + for _, candidate := range candidates { + if !candidate.Compatible || !candidate.Routable { + continue + } + if codexRoutePolicyCapacity(candidate).State != CapacityZero { + exhausted = false + break + } + exhausted = true + } + if exhausted { + return plan, &CachedUsageLimitError{RequestedModel: input.Requirements.RequestedModel} + } + } if input.BoundAccountKey != "" { if err := codexFrozenDispatchContextError(ctx); err != nil { return CodexFrozenDispatchPlan{status: CodexRoutePlanCanceled}, err diff --git a/internal/proxy/codex_frozen_dispatch_plan_test.go b/internal/proxy/codex_frozen_dispatch_plan_test.go index c004124..7a84392 100644 --- a/internal/proxy/codex_frozen_dispatch_plan_test.go +++ b/internal/proxy/codex_frozen_dispatch_plan_test.go @@ -97,6 +97,85 @@ func TestBuildCodexFrozenDispatchPlanUsesProvisionalCounts(t *testing.T) { } } +func TestBuildCodexFrozenDispatchPlanClassifiesExhaustedCapacity(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + remaining []int + defaultKey codex.AccountKey + boundKey codex.AccountKey + unroutable bool + incompatible bool + excluded bool + modelZero bool + wantLimit bool + wantAccounts int + }{ + {name: "all exhausted", remaining: []int{0, 0}, wantLimit: true}, + {name: "default outside pool", remaining: []int{0, 0}, defaultKey: "outside", wantLimit: true}, + {name: "positive alternate", remaining: []int{0, 10}, wantAccounts: 1}, + {name: "unknown alternate", remaining: []int{0, -1}, wantAccounts: 1}, + {name: "required model bucket exhausted", remaining: []int{-1, 10}, modelZero: true, wantLimit: true}, + {name: "empty inventory"}, + {name: "credentials unavailable", remaining: []int{0, 0}, unroutable: true}, + {name: "model incompatible", remaining: []int{0, 0}, incompatible: true}, + {name: "accounts excluded", remaining: []int{0, 0}, excluded: true}, + {name: "terminal default probe", remaining: []int{0, 0}, defaultKey: "account-a", wantAccounts: 1}, + {name: "bound probe", remaining: []int{0, 0}, boundKey: "account-a", wantAccounts: 1}, + {name: "missing bound account", remaining: []int{0, 0}, boundKey: "outside"}, + } { + t.Run(test.name, func(t *testing.T) { + now := time.Unix(1_700_000_000, 0) + capacity := NewCodexCapacityLedger(func() time.Time { return now }, time.Hour) + input := CodexFrozenDispatchInput{ + Capacity: capacity, Requirements: CodexRouteRequirements{RequestedModel: "gpt-5"}, + DefaultAccountKey: test.defaultKey, BoundAccountKey: test.boundKey, Now: now, + } + if test.incompatible { + input.Requirements.RequestedModel = "" + } + if test.modelZero { + input.Requirements.RequiredModels = []string{codexSparkModel} + } + for index, remaining := range test.remaining { + key := []codex.AccountKey{"account-a", "account-b"}[index] + account := frozenDispatchTestLogicalAccount(key, + frozenDispatchCandidate(key, "candidate", "revision", codex.SourceSystem, false, now.Add(time.Hour))) + account.Routable = !test.unroutable + input.Inventory.Accounts = append(input.Inventory.Accounts, account) + if remaining >= 0 { + frozenDispatchObserveCapacity(t, capacity, key, CapacityBucketBase, remaining, now) + } + if test.modelZero { + frozenDispatchObserveCapacity(t, capacity, key, CapacityBucketForModel(codexSparkModel), 0, now) + } + if test.excluded { + input.UnavailableAccountKeys = append(input.UnavailableAccountKeys, key) + } + } + + plan, err := BuildCodexFrozenDispatchPlan(context.Background(), input) + if got := len(plan.Accounts()); got != test.wantAccounts { + t.Fatalf("dispatch accounts = %d, want %d (error %v)", got, test.wantAccounts, err) + } + if test.wantAccounts > 0 && err != nil { + t.Fatalf("available dispatch rejected: %v", err) + } + if err == nil && test.wantAccounts == 0 { + err = plan.TerminalError() + if err == nil { + t.Fatal("empty dispatch did not report a failure") + } + } + var limit *CachedUsageLimitError + if got := errors.As(err, &limit); got != test.wantLimit { + t.Fatalf("capacity error = %t, want %t (error %v)", got, test.wantLimit, err) + } + }) + } +} + func TestBuildCodexFrozenDispatchPlanExcludesDurablyUnavailableAccounts(t *testing.T) { now := time.Unix(1_700_000_000, 0) plan, err := BuildCodexFrozenDispatchPlan(context.Background(), CodexFrozenDispatchInput{ diff --git a/internal/proxy/codex_http_request_plan_test.go b/internal/proxy/codex_http_request_plan_test.go index 1d238f0..77311b9 100644 --- a/internal/proxy/codex_http_request_plan_test.go +++ b/internal/proxy/codex_http_request_plan_test.go @@ -1634,6 +1634,77 @@ func TestCodexHTTPRequestPlanFactoryRoutesFreshAuthenticatedCallerWithinSessionP } } +func TestCodexNativeHTTPExhaustedSessionPoolReturns429(t *testing.T) { + t.Parallel() + + for _, kind := range []CodexRequestKind{CodexRequestTurn, CodexRequestCompaction} { + for _, defaultKey := range []codex.AccountKey{"", "outside"} { + t.Run(string(kind)+"/default="+string(defaultKey), func(t *testing.T) { + now := time.Unix(1_700_000_000, 0).UTC() + capacity := NewCodexCapacityLedger(func() time.Time { return now }, time.Hour) + inventory := codex.Inventory{} + for _, key := range []codex.AccountKey{"outside", "pool-a", "pool-b"} { + inventory.Accounts = append(inventory.Accounts, frozenDispatchTestLogicalAccount(key, + frozenDispatchCandidate(key, "candidate", "revision", codex.SourceSystem, false, now.Add(time.Hour)))) + remaining := 0 + if key == "outside" { + remaining = 100 + } + frozenDispatchObserveCapacity(t, capacity, key, CapacityBucketBase, remaining, now) + } + key := []byte("01234567890123456789012345678901") + runtime := &codexHTTPRequestPlanTestRuntime{} + factory := &CodexHTTPRequestPlanFactory{ + Inventory: &codexHTTPRequestPlanTestInventory{inventory: inventory}, + Capacity: capacity, + Routes: &codexHTTPRequestPlanTestSnapshotter{snapshot: CodexLeaseRouteSnapshot{ + JournalGeneration: 1, + }}, + Runtime: runtime, DefaultAccountKey: defaultKey, + Authority: CodexLeaseAuthorityPolicy{ModeEpoch: 1, Authoritative: true}, + Now: func() time.Time { return now }, + SessionPolicy: NewSessionPolicyResolver(key, routingPolicyV2ForTest(RoutingPolicyV1{ + SchemaVersion: 1, AuthorityGeneration: 1, RoutingGeneration: 7, EffectiveGeneration: 1, + Pools: []AccountPoolV1{{Name: "Cyber", Members: []codex.AccountKey{"pool-a", "pool-b"}}}, + SessionBindings: []SessionBindingV1{{SessionDigest: keyedSessionDigest(key, []byte("session")), Pool: "Cyber"}}, + })), + } + dispatcher := &codexNativeHTTPDispatcher{} + handler, err := NewCodexNativeHTTPHandler(factory, &CodexHTTPRequestSession{Executor: dispatcher}, "https://codex.example/") + if err != nil { + t.Fatal(err) + } + ctx := withRuntimeCallerAuthority(context.Background(), RuntimeCallerAuthorityV1{ + Domain: NormalCallerCodex, SubjectID: "outside", ConsumptionDigest: strings.Repeat("a", 64), + }) + ctx = withRuntimeCallerIdentity(ctx, "outside\x00candidate\x00revision") + path := "/responses" + if kind == CodexRequestCompaction { + path += "/compact" + } + request := httptest.NewRequest(http.MethodPost, path, bytes.NewReader(frozenRequestBody("gpt-5", kind, "body"))).WithContext(ctx) + writer := httptest.NewRecorder() + handler.TryServe(writer, request, kind == CodexRequestCompaction) + + if writer.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429; body = %s", writer.Code, writer.Body.String()) + } + var body struct { + Error struct { + Type string `json:"type"` + } `json:"error"` + } + if err := json.Unmarshal(writer.Body.Bytes(), &body); err != nil || body.Error.Type != "usage_limit_reached" { + t.Fatalf("error response = %s (decode error %v)", writer.Body.String(), err) + } + if runtime.calls != 0 || dispatcher.calls != 0 { + t.Fatalf("exhausted pool began %d leases and %d upstream calls", runtime.calls, dispatcher.calls) + } + }) + } + } +} + func TestCodexHTTPRequestPlanFactoryRejectsUnverifiedCallerContinuity(t *testing.T) { t.Parallel() factory := codexHTTPRequestPlanTestFactory(&codexHTTPRequestPlanTestRuntime{})