From 754e093de76453f74ef4f58fbc66bd7d0fa3fd71 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 26 Aug 2026 12:43:11 +0200 Subject: [PATCH 01/19] feat(scale-set): add service controller --- .github/dependabot.yml | 16 + .github/workflows/lambda.yml | 26 + .github/workflows/release.yml | 60 +- docs/security.md | 10 +- .../compute-providers/aws/ec2/scale-set.ts | 32 + .../aws/ec2/src/runners.d.ts | 3 + .../aws/ec2/src/runners.test.ts | 42 + .../compute-providers/aws/ec2/src/runners.ts | 16 + .../aws/ec2/src/scale-set/provider.test.ts | 781 +++++++++++ .../aws/ec2/src/scale-set/provider.ts | 1205 +++++++++++++++++ lambdas/libs/compute-providers/core/index.ts | 2 +- lambdas/libs/compute-providers/package.json | 3 + .../providers.config.scale-set.ts | 5 + .../libs/compute-providers/scale-set.test.ts | 125 ++ lambdas/libs/compute-providers/scale-set.ts | 209 +++ .../LICENSE.actions-scaleset | 21 + .../libs/github-actions-scale-set/README.md | 69 + .../github-actions-scale-set/package.json | 35 + .../src/client.test.ts | 437 ++++++ .../github-actions-scale-set/src/client.ts | 666 +++++++++ .../src/config.test.ts | 18 + .../github-actions-scale-set/src/config.ts | 117 ++ .../github-actions-scale-set/src/endpoints.ts | 4 + .../github-actions-scale-set/src/errors.ts | 171 +++ .../github-actions-scale-set/src/http.test.ts | 238 ++++ .../libs/github-actions-scale-set/src/http.ts | 327 +++++ .../github-actions-scale-set/src/index.ts | 29 + .../src/message-session-client.test.ts | 281 ++++ .../src/message-session-client.ts | 467 +++++++ .../github-actions-scale-set/src/types.ts | 198 +++ .../github-actions-scale-set/tsconfig.json | 8 + .../github-actions-scale-set/vitest.config.ts | 22 + lambdas/package.json | 3 +- lambdas/services/scale-set/Dockerfile | 25 + .../scale-set/Dockerfile.dockerignore | 7 + lambdas/services/scale-set/README.md | 108 ++ lambdas/services/scale-set/healthcheck.cjs | 13 + lambdas/services/scale-set/package.json | 42 + lambdas/services/scale-set/src/config.test.ts | 184 +++ lambdas/services/scale-set/src/config.ts | 479 +++++++ lambdas/services/scale-set/src/controller.ts | 47 + .../scale-set/src/credentials.test.ts | 111 ++ lambdas/services/scale-set/src/credentials.ts | 125 ++ .../scale-set/src/github-http.test.ts | 58 + lambdas/services/scale-set/src/github-http.ts | 34 + .../scale-set/src/health-server.test.ts | 15 + .../services/scale-set/src/health-server.ts | 53 + lambdas/services/scale-set/src/health.test.ts | 46 + lambdas/services/scale-set/src/health.ts | 137 ++ lambdas/services/scale-set/src/index.ts | 9 + .../services/scale-set/src/lifecycle.test.ts | 45 + lambdas/services/scale-set/src/lifecycle.ts | 51 + lambdas/services/scale-set/src/logger.test.ts | 26 + lambdas/services/scale-set/src/logger.ts | 57 + lambdas/services/scale-set/src/main.ts | 85 ++ .../scale-set/src/parameter-store.test.ts | 78 ++ .../services/scale-set/src/parameter-store.ts | 131 ++ .../services/scale-set/src/reconciler.test.ts | 355 +++++ lambdas/services/scale-set/src/reconciler.ts | 599 ++++++++ lambdas/services/scale-set/tsconfig.json | 8 + lambdas/services/scale-set/vitest.config.ts | 18 + lambdas/yarn.lock | 34 + 62 files changed, 8619 insertions(+), 7 deletions(-) create mode 100644 lambdas/libs/compute-providers/aws/ec2/scale-set.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts create mode 100644 lambdas/libs/compute-providers/providers.config.scale-set.ts create mode 100644 lambdas/libs/compute-providers/scale-set.test.ts create mode 100644 lambdas/libs/compute-providers/scale-set.ts create mode 100644 lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset create mode 100644 lambdas/libs/github-actions-scale-set/README.md create mode 100644 lambdas/libs/github-actions-scale-set/package.json create mode 100644 lambdas/libs/github-actions-scale-set/src/client.test.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/client.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/config.test.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/config.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/endpoints.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/errors.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/http.test.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/http.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/index.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/message-session-client.ts create mode 100644 lambdas/libs/github-actions-scale-set/src/types.ts create mode 100644 lambdas/libs/github-actions-scale-set/tsconfig.json create mode 100644 lambdas/libs/github-actions-scale-set/vitest.config.ts create mode 100644 lambdas/services/scale-set/Dockerfile create mode 100644 lambdas/services/scale-set/Dockerfile.dockerignore create mode 100644 lambdas/services/scale-set/README.md create mode 100644 lambdas/services/scale-set/healthcheck.cjs create mode 100644 lambdas/services/scale-set/package.json create mode 100644 lambdas/services/scale-set/src/config.test.ts create mode 100644 lambdas/services/scale-set/src/config.ts create mode 100644 lambdas/services/scale-set/src/controller.ts create mode 100644 lambdas/services/scale-set/src/credentials.test.ts create mode 100644 lambdas/services/scale-set/src/credentials.ts create mode 100644 lambdas/services/scale-set/src/github-http.test.ts create mode 100644 lambdas/services/scale-set/src/github-http.ts create mode 100644 lambdas/services/scale-set/src/health-server.test.ts create mode 100644 lambdas/services/scale-set/src/health-server.ts create mode 100644 lambdas/services/scale-set/src/health.test.ts create mode 100644 lambdas/services/scale-set/src/health.ts create mode 100644 lambdas/services/scale-set/src/index.ts create mode 100644 lambdas/services/scale-set/src/lifecycle.test.ts create mode 100644 lambdas/services/scale-set/src/lifecycle.ts create mode 100644 lambdas/services/scale-set/src/logger.test.ts create mode 100644 lambdas/services/scale-set/src/logger.ts create mode 100644 lambdas/services/scale-set/src/main.ts create mode 100644 lambdas/services/scale-set/src/parameter-store.test.ts create mode 100644 lambdas/services/scale-set/src/parameter-store.ts create mode 100644 lambdas/services/scale-set/src/reconciler.test.ts create mode 100644 lambdas/services/scale-set/src/reconciler.ts create mode 100644 lambdas/services/scale-set/tsconfig.json create mode 100644 lambdas/services/scale-set/vitest.config.ts diff --git a/.github/dependabot.yml b/.github/dependabot.yml index dd7c872c5a..92b46025d6 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -92,6 +92,22 @@ updates: - dependency-name: "mcr.microsoft.com/vscode/devcontainers/typescript-node" update-types: ["version-update:semver-major"] + - package-ecosystem: "docker" + directory: "/lambdas/services/scale-set" + schedule: + interval: "weekly" + cooldown: + default-days: 7 + labels: + - "dependencies" + - "docker" + commit-message: + prefix: "chore(scale-set)" + # Keep the service runtime aligned with the supported Node.js major. + ignore: + - dependency-name: "node" + update-types: ["version-update:semver-major"] + - package-ecosystem: "pip" directory: "/.github/workflows/mkdocs" schedule: diff --git a/.github/workflows/lambda.yml b/.github/workflows/lambda.yml index 8eba8f46aa..1a270a4461 100644 --- a/.github/workflows/lambda.yml +++ b/.github/workflows/lambda.yml @@ -50,3 +50,29 @@ jobs: name: coverage-reports path: ./**/coverage retention-days: 5 + + scale-set-container: + name: Build scale-set service container + runs-on: ubuntu-latest + steps: + - name: Harden the runner (Audit all outbound calls) + uses: step-security/harden-runner@9af89fc71515a100421586dfdb3dc9c984fbf411 # v2.19.4 + with: + egress-policy: audit + + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Set up QEMU + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + - name: Build scale-set service image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64,linux/arm64 + push: false + cache-from: type=gha,scope=scale-set-service + cache-to: type=gha,mode=max,scope=scale-set-service diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8fcb90d72e..188cd35d20 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,7 @@ name: Release build +env: + SCALE_SET_IMAGE: ghcr.io/${{ github.repository_owner }}/terraform-aws-github-runner-scale-set-service + on: push: branches: @@ -21,6 +24,8 @@ jobs: actions: write # for release-please-action to trigger other workflows id-token: write # for actions/attest-build-provenance to generate attestations attestations: write # for actions/attest-build-provenance to write attestations + artifact-metadata: write # for publishing linked container attestations + packages: write # for publishing the scale-set service image to GHCR environment: release steps: - name: Harden the runner (Audit all outbound calls) @@ -55,6 +60,47 @@ jobs: target-branch: ${{ steps.branch.outputs.name }} release-type: terraform-module token: ${{ steps.token.outputs.token }} + - name: Set up QEMU + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + - name: Set up Docker Buildx + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4 + - name: Log in to the GitHub Container Registry + if: ${{ steps.release.outputs.releases_created == 'true' }} + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and publish scale-set service image + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: scale-set-image + uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7 + with: + context: . + file: ./lambdas/services/scale-set/Dockerfile + platforms: linux/amd64,linux/arm64 + push: true + tags: | + ${{ env.SCALE_SET_IMAGE }}:${{ steps.release.outputs.tag_name }} + ${{ env.SCALE_SET_IMAGE }}:latest + labels: | + org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.version=${{ steps.release.outputs.tag_name }} + sbom: true + provenance: mode=max + cache-from: type=gha,scope=scale-set-service + cache-to: type=gha,mode=max,scope=scale-set-service + - name: Attest scale-set service image + if: ${{ steps.release.outputs.releases_created == 'true' }} + id: scale-set-image-attest + uses: actions/attest@1e69f48acb82d1966a394da916b4c1698aa569d6 # v4 + with: + subject-name: ${{ env.SCALE_SET_IMAGE }} + subject-digest: ${{ steps.scale-set-image.outputs.digest }} + push-to-registry: true - name: Attest if: ${{ steps.release.outputs.releases_created == 'true' }} id: attest @@ -65,20 +111,26 @@ jobs: if: ${{ steps.release.outputs.releases_created == 'true' }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - VERSION: ${{ github.event.inputs.version }} TAG_NAME: ${{ steps.release.outputs.tag_name }} ATTESTATION_URL: ${{ steps.attest.outputs.attestation-url }} + CONTAINER_ATTESTATION_URL: ${{ steps.scale-set-image-attest.outputs.attestation-url }} + CONTAINER_IMAGE: ${{ env.SCALE_SET_IMAGE }} + CONTAINER_DIGEST: ${{ steps.scale-set-image.outputs.digest }} REPOSITORY: ${{ github.repository }} run: | - version="${VERSION}" tag_name="${TAG_NAME}" attestation_url="${ATTESTATION_URL}" + container_attestation_url="${CONTAINER_ATTESTATION_URL}" + container_image="${CONTAINER_IMAGE}" + container_digest="${CONTAINER_DIGEST}" repository="${REPOSITORY}" - gh release view $version --json body -q '.body' > new-release-notes.md + gh release view "$tag_name" --json body -q '.body' > new-release-notes.md echo "## Attestation" >> new-release-notes.md echo "Attestation url: $attestation_url" >> new-release-notes.md echo "Verify the artifacts by running \`gh attestation verify --repo ${repository}\`" >> new-release-notes.md - gh release edit $tag_name -F new-release-notes.md -t $tag_name + echo "Scale-set service image: \`${container_image}@${container_digest}\`" >> new-release-notes.md + echo "Container attestation url: $container_attestation_url" >> new-release-notes.md + gh release edit "$tag_name" -F new-release-notes.md -t "$tag_name" - name: Upload release assets if: ${{ steps.release.outputs.releases_created == 'true' }} env: diff --git a/docs/security.md b/docs/security.md index a94688b234..4ef4d17b94 100644 --- a/docs/security.md +++ b/docs/security.md @@ -14,6 +14,14 @@ The examples are using standard AMI's for different operating systems. Instances ## Attestation -The module is released using GitHub actions and the lambda artifacts are attached to the release as attachment. During the release attestations are created. The attestations are created by the release pipeline. You find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release. The attestations are not a security guarantee. We recommend you to verify the attestation after downloading the lambda artifacts. +The module is released using GitHub Actions and the Lambda artifacts are attached to the release. The release pipeline creates provenance attestations for those artifacts. You can find a link to the attestation in the GitHub release. The attestation only provides provenance information about the release; it is not a security guarantee. We recommend verifying the attestation after downloading the Lambda artifacts. + +Releases also publish the multi-architecture scale-set service image to the GitHub Container Registry with an SBOM, build provenance, and a registry attestation. The convenience image default follows the latest module release. Production deployments should override it with the immutable image digest printed in the release notes, then verify that image with: + +```bash +gh attestation verify \ + oci://ghcr.io/github-aws-runners/terraform-aws-github-runner-scale-set-service@sha256: \ + --repo github-aws-runners/terraform-aws-github-runner +``` --8<-- "SECURITY.md:mkdocsrunners" diff --git a/lambdas/libs/compute-providers/aws/ec2/scale-set.ts b/lambdas/libs/compute-providers/aws/ec2/scale-set.ts new file mode 100644 index 0000000000..671ddc69f7 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/scale-set.ts @@ -0,0 +1,32 @@ +import type { ScaleSetComputeProviderModule, ScaleSetComputeProviderPlugin } from '../../scale-set'; + +import { createEc2ScaleSetProvider, type Ec2ScaleSetProviderDependencies } from './src/scale-set/provider'; + +export type { Ec2ScaleSetProviderConfig, Ec2ScaleSetProviderDependencies } from './src/scale-set/provider'; +export { createEc2ScaleSetProvider, parseEc2ScaleSetProviderConfig } from './src/scale-set/provider'; + +export function createEc2ScaleSetPlugin( + dependencies: Ec2ScaleSetProviderDependencies = {}, +): ScaleSetComputeProviderPlugin<'ec2'> { + return { + type: 'ec2', + capabilities: { + environmentVariables: {}, + create: ({ runnerConfigName, scaleSetId, githubScope, configuration }) => + createEc2ScaleSetProvider( + { + runnerConfigName, + scaleSetId, + githubScope, + configuration: configuration as Parameters[0]['configuration'], + }, + dependencies, + ), + }, + }; +} + +export const provider = { + type: 'ec2', + createPlugin: createEc2ScaleSetPlugin, +} satisfies ScaleSetComputeProviderModule<'ec2'>; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts index e711ed5318..51b5432270 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts @@ -6,6 +6,7 @@ import { _InstanceType, Placement, FleetBlockDeviceMappingRequest, + type Tag, } from '@aws-sdk/client-ec2'; import type { ListRunnerFilters, RunnerSource, RunnerType } from '../../../core'; @@ -47,4 +48,6 @@ export interface RunnerInputParameters { tracingEnabled?: boolean; onDemandFailoverOnError?: string[]; useDedicatedHost?: boolean; + /** Orchestrator-owned tags applied to instances, volumes, and fleets. */ + orchestrationTags?: readonly Tag[]; } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts index dd80e7b4dd..6699f0399c 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts @@ -423,6 +423,29 @@ describe('create runner', () => { }); }); + it.each([ + ['a provider-owned tag', [{ Key: 'ghr:Owner', Value: 'another-owner' }]], + [ + 'a duplicate orchestration tag', + [ + { Key: 'ghr:scale_set_id', Value: '42' }, + { Key: 'ghr:scale_set_id', Value: '43' }, + ], + ], + ])('rejects %s before creating a Fleet', async (_description, orchestrationTags) => { + await expect( + ec2Operations.create({ + ...createRunnerConfig(defaultRunnerConfig), + orchestrationTags, + }), + ).resolves.toEqual({ + instances: [], + failedInstanceCount: 1, + failureCodes: ['aws-name:Error'], + }); + expect(mockEC2Client).not.toHaveReceivedCommand(CreateFleetCommand); + }); + it('calls create fleet of 1 instance with the on-demand capacity', async () => { await ec2Operations.create( createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand', allocationStrategy: 'lowest-price' }), @@ -1493,6 +1516,25 @@ describe('create runner with useDedicatedHost', () => { }); }); + it('passes orchestration tags to RunInstances resources', async () => { + const orchestrationTags = [ + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:scale_set_id', Value: '42' }, + ]; + + await ec2Operations.create({ + ...createRunnerConfig(dedicatedHostRunnerConfig), + orchestrationTags, + }); + + expect(mockEC2Client).toHaveReceivedCommandWith(RunInstancesCommand, { + TagSpecifications: [ + { ResourceType: 'instance', Tags: expect.arrayContaining(orchestrationTags) }, + { ResourceType: 'volume', Tags: expect.arrayContaining(orchestrationTags) }, + ], + }); + }); + it('creates multiple instances via RunInstances and preserves the caller source', async () => { mockEC2Client.on(RunInstancesCommand).resolves({ Instances: [{ InstanceId: 'i-dedicated-1' }, { InstanceId: 'i-dedicated-2' }], diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.ts index d746888a6f..ac16551a87 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.ts @@ -27,12 +27,24 @@ import type { Ec2RunnerCreateResult, Ec2RunnerFailureCode } from './runner-creat import type { Ec2ListRunnerFilters, Ec2OverrideConfig, RunnerInputParameters } from './runners.d'; const logger = createChildLogger('runners'); +const BASE_RUNNER_TAG_KEYS = new Set(['ghr:Application', 'ghr:created_by', 'ghr:Type', 'ghr:Owner', 'ghr:trace_id']); interface Ec2Filter { Name: string; Values: string[]; } +function appendOrchestrationRunnerTags(tags: Tag[], orchestrationTags: readonly Tag[] | undefined): void { + const keys = new Set(BASE_RUNNER_TAG_KEYS); + for (const tag of orchestrationTags ?? []) { + if (!tag.Key || tag.Value === undefined || keys.has(tag.Key)) { + throw new Error(`Orchestration runner tag '${tag.Key ?? ''}' is invalid or duplicates a provider-owned tag`); + } + keys.add(tag.Key); + tags.push({ Key: tag.Key, Value: tag.Value }); + } +} + export interface Ec2RunnerRequestContext { readonly signal: AbortSignal | undefined; } @@ -551,6 +563,8 @@ async function createInstances( { Key: 'ghr:Type', Value: runnerParameters.runnerType }, { Key: 'ghr:Owner', Value: runnerParameters.runnerOwner }, ]; + appendOrchestrationRunnerTags(tags, runnerParameters.orchestrationTags); + if (runnerParameters.tracingEnabled) { const traceId = tracer.getRootXrayTraceId(); tags.push({ Key: 'ghr:trace_id', Value: traceId! }); @@ -635,6 +649,8 @@ async function createInstancesWithRunInstances( { Key: 'ghr:Type', Value: runnerParameters.runnerType }, { Key: 'ghr:Owner', Value: runnerParameters.runnerOwner }, ]; + appendOrchestrationRunnerTags(tags, runnerParameters.orchestrationTags); + if (runnerParameters.tracingEnabled) { const traceId = tracer.getRootXrayTraceId(); tags.push({ Key: 'ghr:trace_id', Value: traceId! }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts new file mode 100644 index 0000000000..7e10f010d9 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts @@ -0,0 +1,781 @@ +import { createHash } from 'node:crypto'; + +import { + CreateFleetCommand, + CreateTagsCommand, + DescribeInstancesCommand, + EC2Client, + TerminateInstancesCommand, + type Instance, +} from '@aws-sdk/client-ec2'; +import { DeleteParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import type { + GenerateScaleSetJitConfigurationResult, + ScaleSetReconcileRequest, + ScaleSetRunnerState, +} from '../../../../scale-set'; +import { + createEc2ScaleSetProvider, + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_GITHUB_RUNNER_ID_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, + parseEc2ScaleSetProviderConfig, + type Ec2ScaleSetProviderConfig, +} from './provider'; + +const ec2Mock = mockClient(EC2Client); +const ssmMock = mockClient(SSMClient); +const ec2Client = new EC2Client({ region: 'eu-west-1' }); +const ssmClient = new SSMClient({ region: 'eu-west-1' }); +const signal = new AbortController().signal; +const githubScope = 'https://github.com/example'; +const githubScopeHash = createHash('sha256').update(githubScope, 'utf8').digest('hex'); + +const config: Ec2ScaleSetProviderConfig = { + region: 'eu-west-1', + environment: 'unit-test', + runnerNamePrefix: 'runner-', + jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', + subnets: ['subnet-12345678'], + launchTemplateName: 'unit-test-runners', + ec2instanceCriteria: { + instanceTypes: ['m7i.large'], + targetCapacityType: 'on-demand', + instanceAllocationStrategy: 'lowest-price', + }, + scaleErrors: ['InsufficientInstanceCapacity'], + ssmParameterTags: [{ Key: 'Project', Value: 'runner-tests' }], +}; + +function ownedInstance( + instanceId: string, + identity?: { runnerId: number; runnerName: string }, + overrides: { + runnerConfigName?: string; + scaleSetId?: number; + scaleSetState?: string; + githubScopeHash?: string; + launchTime?: Date; + } = {}, +): Instance { + return { + InstanceId: instanceId, + LaunchTime: overrides.launchTime ?? new Date('2026-08-24T10:00:00Z'), + Tags: [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: overrides.runnerConfigName ?? 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(overrides.scaleSetId ?? 42) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: overrides.githubScopeHash ?? githubScopeHash }, + { + Key: EC2_SCALE_SET_STATE_TAG, + Value: overrides.scaleSetState ?? (identity ? 'config-published' : 'provisioning'), + }, + ...(identity + ? [ + { Key: EC2_RUNNER_NAME_TAG, Value: identity.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(identity.runnerId) }, + ] + : []), + ], + }; +} + +function githubState( + runnerId: number, + runnerName: string, + overrides: Partial = {}, +): ScaleSetRunnerState { + return { + runnerId, + runnerName, + scaleSetId: 42, + status: 'online', + busy: false, + lifecycle: 'unknown', + ...overrides, + }; +} + +function jitResult(instanceId = 'i-1234567890abcdef0'): GenerateScaleSetJitConfigurationResult { + return { + encodedJitConfiguration: 'sensitive-encoded-jit-configuration', + runnerId: 101, + runnerName: `runner-${instanceId}`, + scaleSetId: 42, + }; +} + +function createRequest(overrides: Partial = {}): ScaleSetReconcileRequest { + return { + desiredRunners: 1, + bootTimeoutMinutes: 10, + runnerInventoryComplete: false, + runnerStates: [], + signal, + generateJitConfiguration: vi.fn().mockResolvedValue(jitResult()), + removeRunner: vi.fn().mockResolvedValue({ status: 'removed' }), + ...overrides, + }; +} + +function provider(options: { githubScope?: string; now?: () => number } = {}) { + return createEc2ScaleSetProvider( + { + runnerConfigName: 'linux', + scaleSetId: 42, + githubScope: options.githubScope ?? githubScope, + configuration: config, + }, + { + ec2Client, + ssmClient, + now: options.now ?? (() => new Date('2026-08-24T10:05:00Z').getTime()), + }, + ); +} + +beforeEach(() => { + ec2Mock.reset(); + ssmMock.reset(); + ec2Mock.on(CreateTagsCommand).resolves({}); + ec2Mock.on(TerminateInstancesCommand).resolves({}); + ssmMock.on(PutParameterCommand).resolves({}); + ssmMock.on(DeleteParameterCommand).resolves({}); +}); + +describe('EC2 scale-set provider configuration', () => { + it('strictly parses the supported provider-owned configuration', () => { + expect(parseEc2ScaleSetProviderConfig(config)).toMatchObject(config); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: '' })).toMatchObject({ + runnerNamePrefix: '', + }); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: 'r'.repeat(45) })).toMatchObject({ + runnerNamePrefix: 'r'.repeat(45), + }); + }); + + it.each([ + [{ ...config, region: '$(credential)' }], + [{ ...config, subnets: ['subnet-12345678', 'subnet-12345678'] }], + [{ ...config, ec2instanceCriteria: { ...config.ec2instanceCriteria, instanceAllocationStrategy: 'diversified' } }], + [{ ...config, ec2OverrideConfig: { UserData: 'untrusted' } }], + [{ ...config, ssmParameterTags: [{ Key: 'aws:owner', Value: 'untrusted' }] }], + [{ ...config, runnerNamePrefix: 'r'.repeat(46) }], + [{ ...config, bootTimeoutMinutes: 10 }], + ])('rejects invalid or unsupported values instead of forwarding them to AWS', (invalid) => { + expect(() => parseEc2ScaleSetProviderConfig(invalid)).toThrow(); + }); + + it('does not expose configurable EC2 ownership or lifecycle tags', () => { + expect(Object.keys(config)).not.toContain('additionalTags'); + expect(() => + parseEc2ScaleSetProviderConfig({ + ...config, + additionalTags: [{ Key: EC2_SCALE_SET_ID_TAG, Value: 'another-scale-set' }], + }), + ).toThrow("Unsupported EC2 scale-set configuration field 'configuration.additionalTags'"); + }); + + it('rejects non-canonical GitHub ownership scopes before creating clients', () => { + expect(() => provider({ githubScope: 'https://GITHUB.com/example/' })).toThrow( + 'githubScope must be a canonical HTTPS GitHub configuration URL', + ); + }); +}); + +describe('EC2 scale-set reconciliation', () => { + it('lists only the exact runner-config and scale-set ownership boundary', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ + ownedInstance('i-owned', { runnerId: 101, runnerName: 'runner-i-owned' }), + ownedInstance('i-other', undefined, { runnerConfigName: 'other' }), + ownedInstance( + 'i-other-scope', + { runnerId: 102, runnerName: 'runner-i-other-scope' }, + { + githubScopeHash: createHash('sha256').update('https://github.com/another', 'utf8').digest('hex'), + }, + ), + ], + }, + ], + }); + + const result = await provider().reconcile(createRequest()); + + expect(result).toMatchObject({ status: 'converged', desiredRunners: 1, currentRunners: 1 }); + expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { + Filters: expect.arrayContaining([ + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: ['linux'] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: ['42'] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash] }, + ]), + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts a young handed-off instance as serving during its bounded boot window', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-booting', { runnerId: 101, runnerName: 'runner-i-booting' })], + }, + ], + }); + + const result = await provider({ now: () => new Date('2026-08-24T10:09:59Z').getTime() }).reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('uses the orchestration request boot window instead of provider configuration', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-at-timeout', { runnerId: 101, runnerName: 'runner-i-at-timeout' })], + }, + ], + }); + + const result = await provider({ now: () => new Date('2026-08-24T10:05:00Z').getTime() }).reconcile( + createRequest({ bootTimeoutMinutes: 5 }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { launched: 0, retainedUnknown: 1 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('requests a complete inventory for an old handoff, then counts only its exact online identity', async () => { + const instance = ownedInstance('i-old', { runnerId: 101, runnerName: 'runner-i-old' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const computeProvider = provider({ now: () => new Date('2026-08-24T10:10:00Z').getTime() }); + + const firstPass = await computeProvider.reconcile(createRequest()); + + expect(firstPass).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { launched: 0, retainedUnknown: 1 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + + const secondPass = await computeProvider.reconcile( + createRequest({ + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-i-old', { status: 'online', lifecycle: 'unknown' })], + }), + ); + + expect(secondPass).toMatchObject({ + status: 'converged', + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts an exact JobStarted identity as serving without waiting for public inventory', async () => { + const instance = ownedInstance('i-started', { runnerId: 101, runnerName: 'runner-i-started' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + + const result = await provider({ now: () => new Date('2026-08-24T12:00:00Z').getTime() }).reconcile( + createRequest({ + runnerStates: [ + githubState(101, 'runner-i-started', { status: 'unknown', busy: undefined, lifecycle: 'started' }), + ], + }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 0, retainedUnknown: 0 }, + }); + }); + + it('retains an old offline handoff and bounds replacement to one physical surge instance', async () => { + const old = ownedInstance('i-old-offline', { runnerId: 100, runnerName: 'runner-i-old-offline' }); + const replacementId = 'i-1234567890abcdef0'; + const replacement = ownedInstance( + replacementId, + { runnerId: 101, runnerName: `runner-${replacementId}` }, + { + launchTime: new Date('2026-08-24T10:10:30Z'), + }, + ); + ec2Mock + .on(DescribeInstancesCommand) + .resolvesOnce({ Reservations: [{ Instances: [old] }] }) + .resolves({ Reservations: [{ Instances: [old, replacement] }] }); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacementId] }] }); + const computeProvider = provider({ now: () => new Date('2026-08-24T10:11:00Z').getTime() }); + const completeInventory = createRequest({ + runnerInventoryComplete: true, + runnerStates: [ + githubState(100, 'runner-i-old-offline', { + status: 'offline', + busy: false, + lifecycle: 'completed', + }), + ], + }); + + const result = await computeProvider.reconcile(completeInventory); + const nextResult = await computeProvider.reconcile(completeInventory); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + needsRunnerInventory: false, + actions: { launched: 1, retainedUnknown: 1 }, + }); + expect(nextResult).toMatchObject({ + status: 'retained', + currentRunners: 2, + needsRunnerInventory: false, + actions: { launched: 0, retainedUnknown: 1 }, + }); + expect(ec2Mock).toHaveReceivedCommandTimes(CreateFleetCommand, 1); + }); + + it.each(['provisioning', 'publishing'])( + 'retains interrupted %s capacity but provisions a replacement', + async (scaleSetState) => { + const stuck = ownedInstance( + 'i-stuck', + scaleSetState === 'publishing' ? { runnerId: 100, runnerName: 'runner-i-stuck' } : undefined, + { scaleSetState }, + ); + const replacement = 'i-1234567890abcdef0'; + ec2Mock + .on(DescribeInstancesCommand) + .resolvesOnce({ Reservations: [{ Instances: [stuck] }] }) + .resolves({ + Reservations: [ + { + Instances: [stuck, ownedInstance(replacement, { runnerId: 101, runnerName: `runner-${replacement}` })], + }, + ], + }); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacement] }] }); + const computeProvider = provider(); + + const result = await computeProvider.reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 1, terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-stuck'] }); + expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { + Name: `${config.jitConfigParameterPath}/${replacement}`, + }); + + const nextResult = await computeProvider.reconcile(createRequest()); + expect(nextResult).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).toHaveReceivedCommandTimes(CreateFleetCommand, 1); + }, + ); + + it('caps retained-capacity replacement surge when every replacement remains ambiguous', async () => { + const ambiguous = [ + ownedInstance('i-stuck-1', undefined, { scaleSetState: 'provisioning' }), + ownedInstance('i-stuck-2', { runnerId: 102, runnerName: 'runner-i-stuck-2' }, { scaleSetState: 'publishing' }), + ]; + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: ambiguous }] }); + + const result = await provider().reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedUnknown: 2 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('launches owned compute, verifies JIT identity, and publishes only a SecureString', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const generateJitConfiguration = vi.fn().mockResolvedValue(jitResult(instanceId)); + + const result = await provider().reconcile(createRequest({ generateJitConfiguration })); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 1, + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 1, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }); + expect(generateJitConfiguration).toHaveBeenCalledWith({ runnerName: `runner-${instanceId}`, signal }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateFleetCommand, { + TagSpecifications: expect.arrayContaining([ + expect.objectContaining({ + ResourceType: 'instance', + Tags: expect.arrayContaining([ + { Key: 'ghr:Owner', Value: 'example' }, + { Key: 'ghr:Type', Value: 'Org' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, + ]), + }), + ]), + }); + expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + Value: 'sensitive-encoded-jit-configuration', + Type: 'SecureString', + Overwrite: false, + Tags: expect.arrayContaining([ + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + ]), + }); + }); + + it('does not remove an unrelated GitHub runner when JIT identity validation fails', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const removeRunner = vi.fn(); + + const result = await provider().reconcile( + createRequest({ + generateJitConfiguration: vi.fn().mockResolvedValue({ + ...jitResult(instanceId), + runnerName: 'runner-owned-by-another-config', + }), + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'non_retryable_error', + currentRunners: 0, + actions: { launched: 0, terminated: 1 }, + errors: [expect.objectContaining({ operation: 'generate_jit_configuration', retryable: false })], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ssmMock).not.toHaveReceivedCommand(PutParameterCommand); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: [instanceId] }); + }); + + it('retains compute when failed JIT publication cannot be safely cancelled', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('redacted secret'), { name: 'TimeoutError' })); + ssmMock.on(DeleteParameterCommand).rejects(Object.assign(new Error('missing'), { name: 'ParameterNotFound' })); + const removeRunner = vi.fn(); + + const result = await provider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'retryable_error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + errors: [expect.objectContaining({ operation: 'publish_jit_configuration', code: 'TimeoutError' })], + }); + expect(JSON.stringify(result)).not.toContain('redacted secret'); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not treat a successful DeleteParameter as proof that bootstrap did not read JIT first', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('throttled'), { name: 'ThrottlingException' })); + ssmMock.on(DeleteParameterCommand).resolves({}); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await provider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'retryable_error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + }); + expect(ssmMock).toHaveReceivedCommandWith(DeleteParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('terminates only exact known-idle or completed runners and retains busy or unknown runners', async () => { + const completed = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + const busy = ownedInstance('i-busy', { runnerId: 102, runnerName: 'runner-busy' }); + const unknown = ownedInstance('i-unknown', { runnerId: 103, runnerName: 'runner-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [completed, busy, unknown] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await provider().reconcile( + createRequest({ + desiredRunners: 2, + runnerInventoryComplete: true, + runnerStates: [ + githubState(101, 'runner-completed', { status: 'offline', busy: undefined, lifecycle: 'completed' }), + githubState(102, 'runner-busy', { busy: true, lifecycle: 'started' }), + ], + removeRunner, + }), + ); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 2, + currentRunners: 2, + needsRunnerInventory: false, + actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + expect(removeRunner).toHaveBeenCalledWith({ + runnerId: 101, + runnerName: 'runner-completed', + scaleSetId: 42, + signal, + }); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-completed'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-busy'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-unknown'] }); + }); + + it('uses a typed inventory signal for a conservative first pass and exact second pass', async () => { + const instance = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + const computeProvider = provider(); + + const firstPass = await computeProvider.reconcile( + createRequest({ desiredRunners: 0, runnerStates: [], removeRunner }), + ); + + expect(firstPass).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + + const secondPass = await computeProvider.reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-completed', { lifecycle: 'completed', status: 'offline' })], + removeRunner, + }), + ); + + expect(secondPass).toMatchObject({ + status: 'converged', + currentRunners: 0, + needsRunnerInventory: false, + actions: { terminated: 1 }, + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + }); + + it('never lets a completed lifecycle marker override a current busy signal', async () => { + const instance = ownedInstance('i-completed-busy', { runnerId: 101, runnerName: 'runner-completed-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await provider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [ + githubState(101, 'runner-completed-busy', { + lifecycle: 'completed', + status: 'online', + busy: true, + }), + ], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: false, + actions: { terminated: 0, retainedBusy: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner without an error when the exact removal check observes that it became busy', async () => { + const instance = ownedInstance('i-raced-busy', { runnerId: 101, runnerName: 'runner-raced-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_busy' }); + + const result = await provider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-raced-busy')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: false, + actions: { terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner and requests inventory when exact removal observes identity drift', async () => { + const instance = ownedInstance('i-raced-unknown', { runnerId: 101, runnerName: 'runner-raced-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_unknown' }); + + const result = await provider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-raced-unknown')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: false, + actions: { terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not trust a mutable EC2 GitHub-runner-id tag when controller identity disagrees', async () => { + const instance = ownedInstance('i-mismatch', { runnerId: 999, runnerName: 'runner-exact' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await provider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-exact')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not terminate compute when exact GitHub removal fails', async () => { + const instance = ownedInstance('i-idle', { runnerId: 101, runnerName: 'runner-idle' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi + .fn() + .mockRejectedValue(Object.assign(new Error('must not leak'), { name: 'ServiceUnavailable' })); + + const result = await provider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-idle')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retryable_error', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: expect.arrayContaining([expect.objectContaining({ operation: 'remove_runner', retryable: true })]), + }); + expect(JSON.stringify(result)).not.toContain('must not leak'); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('rejects an invalid desired count without touching AWS', async () => { + const result = await provider().reconcile(createRequest({ desiredRunners: -1 })); + + expect(result).toMatchObject({ + status: 'non_retryable_error', + desiredRunners: -1, + currentRunners: 0, + errors: [{ operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT', retryable: false }], + }); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); + + it.each([0, 121, 1.5])('rejects invalid orchestration boot timeout %s without touching AWS', async (value) => { + const result = await provider().reconcile(createRequest({ bootTimeoutMinutes: value })); + + expect(result).toMatchObject({ + status: 'non_retryable_error', + currentRunners: 0, + errors: [{ operation: 'validate', code: 'INVALID_BOOT_TIMEOUT', retryable: false }], + }); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); + + it('propagates cancellation instead of converting shutdown into a retry result', async () => { + const abort = new AbortController(); + abort.abort(new Error('service stopping')); + + await expect(provider().reconcile(createRequest({ signal: abort.signal }))).rejects.toThrow('service stopping'); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts new file mode 100644 index 0000000000..e775527127 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts @@ -0,0 +1,1205 @@ +import { createHash } from 'node:crypto'; + +import { CreateTagsCommand, DescribeInstancesCommand, EC2Client, type Instance, type Tag } from '@aws-sdk/client-ec2'; +import { + DeleteParameterCommand, + GetParameterCommand, + PutParameterCommand, + SSMClient, + type Tag as SsmTag, +} from '@aws-sdk/client-ssm'; + +import type { + GenerateScaleSetJitConfigurationResult, + ScaleSetComputeProvider, + ScaleSetReconcileActions, + ScaleSetReconcileError, + ScaleSetReconcileOperation, + ScaleSetReconcileRequest, + ScaleSetReconcileResult, + ScaleSetRunnerState, +} from '../../../../scale-set'; +import { createRunner, terminateRunner } from '../runners'; +import type { Ec2OverrideConfig, RunnerInputParameters } from '../runners.d'; + +export const EC2_RUNNER_CONFIG_TAG = 'ghr:runner_config'; +export const EC2_SCALE_SET_ID_TAG = 'ghr:scale_set_id'; +export const EC2_GITHUB_SCOPE_HASH_TAG = 'ghr:github_scope_hash'; +export const EC2_SCALE_SET_STATE_TAG = 'ghr:scale_set_state'; +export const EC2_RUNNER_NAME_TAG = 'ghr:runner_name'; +export const EC2_GITHUB_RUNNER_ID_TAG = 'ghr:github_runner_id'; + +const APPLICATION_TAG = 'ghr:Application'; +const APPLICATION_VALUE = 'github-action-runner'; +const CREATED_BY_TAG = 'ghr:created_by'; +const CREATED_BY_VALUE = 'scale-set-service'; +const ENVIRONMENT_TAG = 'ghr:environment'; +const SSM_STANDARD_TIER_THRESHOLD = 4000; +const SSM_ADVANCED_TIER_MAX_BYTES = 8192; +const GITHUB_RUNNER_NAME_MAX_LENGTH = 64; +const RETAINED_CAPACITY_REPLACEMENT_SURGE = 1; +const MAX_BOOT_TIMEOUT_MINUTES = 120; +const SPOT_ALLOCATION_STRATEGIES = new Set([ + 'lowest-price', + 'diversified', + 'capacity-optimized', + 'capacity-optimized-prioritized', + 'price-capacity-optimized', +]); +const ON_DEMAND_ALLOCATION_STRATEGIES = new Set(['lowest-price', 'prioritized']); + +type Ec2ScaleSetState = 'provisioning' | 'publishing' | 'config-published' | 'retiring'; + +export interface Ec2ScaleSetProviderConfig { + region: string; + environment: string; + runnerNamePrefix: string; + jitConfigParameterPath: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; + ec2OverrideConfig?: Ec2OverrideConfig; + amiIdSsmParameterName?: string; + tracingEnabled?: boolean; + onDemandFailoverOnError?: string[]; + scaleErrors: string[]; + useDedicatedHost?: boolean; + ssmKmsKeyId?: string; + ssmParameterTags?: SsmTag[]; +} + +export interface Ec2ScaleSetProviderDependencies { + ec2Client?: EC2Client; + ssmClient?: SSMClient; + now?: () => number; +} + +export interface CreateEc2ScaleSetProviderInput { + runnerConfigName: string; + scaleSetId: number; + githubScope: string; + configuration: Ec2ScaleSetProviderConfig; +} + +interface OwnedEc2Runner { + instanceId: string; + launchTime?: Date; + githubRunnerId?: number; + runnerName?: string; + scaleSetState?: Ec2ScaleSetState; +} + +interface MutableReconcileState { + currentRunners: number; + needsRunnerInventory: boolean; + retainedUnknownResourceIds: Set; + actions: ScaleSetReconcileActions; + errors: ScaleSetReconcileError[]; +} + +class NonRetryableScaleSetError extends Error { + constructor(message: string) { + super(message); + this.name = 'NonRetryableScaleSetError'; + } +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function rejectUnknownKeys(value: Record, allowedKeys: ReadonlySet, name: string): void { + const unknownKey = Object.keys(value).find((key) => !allowedKeys.has(key)); + if (unknownKey !== undefined) { + throw new NonRetryableScaleSetError(`Unsupported EC2 scale-set configuration field '${name}.${unknownKey}'`); + } +} + +function requireString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length === 0 || value.length > maximumLength || !pattern.test(value)) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requirePossiblyEmptyString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length > maximumLength || !pattern.test(value)) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function optionalString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string | undefined { + if (value === undefined) return undefined; + return requireString(value, name, pattern, maximumLength); +} + +function optionalBoolean(value: unknown, name: string): boolean | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'boolean') { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requireStringArray( + value: unknown, + name: string, + pattern: RegExp, + maximumItemLength: number, + allowEmpty = false, +): string[] { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.length > 100) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + const parsed = value.map((item, index) => requireString(item, `${name}[${index}]`, pattern, maximumItemLength)); + if (new Set(parsed).size !== parsed.length) { + throw new NonRetryableScaleSetError(`EC2 scale-set configuration field '${name}' contains duplicate values`); + } + return parsed; +} + +function parseInstanceTypePriorities(value: unknown): Record | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'instanceTypePriorities'"); + } + + const result = Object.create(null) as Record; + for (const [instanceType, priority] of Object.entries(value)) { + requireString(instanceType, 'instanceTypePriorities key', /^[a-z0-9][a-z0-9.-]*$/, 64); + if (typeof priority !== 'number' || !Number.isSafeInteger(priority) || priority < 0 || priority > 1000) { + throw new NonRetryableScaleSetError( + `Invalid EC2 scale-set configuration priority for instance type '${instanceType}'`, + ); + } + result[instanceType] = priority; + } + return result; +} + +function requireSsmTagValue(value: unknown): string { + if (typeof value !== 'string' || value.length > 256) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + for (const character of value) { + const codePoint = character.codePointAt(0)!; + if (codePoint < 32 || codePoint === 127) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + } + return value; +} + +function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2OverrideConfig'"); + } + + const supportedKeys = new Set([ + 'InstanceType', + 'MaxPrice', + 'SubnetId', + 'AvailabilityZone', + 'AvailabilityZoneId', + 'WeightedCapacity', + 'Priority', + 'ImageId', + ]); + if (Object.keys(value).some((key) => !supportedKeys.has(key))) { + throw new NonRetryableScaleSetError('EC2 scale-set configuration contains an unsupported launch override'); + } + + const weightedCapacity = value.WeightedCapacity; + const priority = value.Priority; + for (const [name, number] of [ + ['WeightedCapacity', weightedCapacity], + ['Priority', priority], + ] as const) { + if (number !== undefined && (typeof number !== 'number' || !Number.isFinite(number) || number < 0)) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + } + + return { + InstanceType: optionalString( + value.InstanceType, + 'ec2OverrideConfig.InstanceType', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ) as Ec2OverrideConfig['InstanceType'], + MaxPrice: optionalString(value.MaxPrice, 'ec2OverrideConfig.MaxPrice', /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, 32), + SubnetId: optionalString(value.SubnetId, 'ec2OverrideConfig.SubnetId', /^subnet-[0-9a-f]+$/, 32), + AvailabilityZone: optionalString( + value.AvailabilityZone, + 'ec2OverrideConfig.AvailabilityZone', + /^[a-z]{2}(?:-[a-z0-9]+)+-\d[a-z]$/, + 64, + ), + AvailabilityZoneId: optionalString( + value.AvailabilityZoneId, + 'ec2OverrideConfig.AvailabilityZoneId', + /^[a-z0-9-]+$/, + 64, + ), + WeightedCapacity: weightedCapacity as number | undefined, + Priority: priority as number | undefined, + ImageId: optionalString(value.ImageId, 'ec2OverrideConfig.ImageId', /^ami-[0-9a-f]+$/, 32), + }; +} + +function parseSsmTags(value: unknown): SsmTag[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.length > 45) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + + const tags: SsmTag[] = []; + const keys = new Set(); + for (const item of value) { + if (!isRecord(item)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + const key = requireString(item.Key, 'ssmParameterTags.Key', /^[A-Za-z0-9_.:/=+@-]+$/, 128); + const tagValue = requireSsmTagValue(item.Value); + if (key.toLowerCase().startsWith('aws:') || keys.has(key)) { + throw new NonRetryableScaleSetError(`Invalid or duplicate SSM tag key '${key}'`); + } + keys.add(key); + tags.push({ Key: key, Value: tagValue }); + } + return tags; +} + +export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProviderConfig { + if (!isRecord(value)) { + throw new NonRetryableScaleSetError('EC2 scale-set provider configuration must be an object'); + } + rejectUnknownKeys( + value, + new Set([ + 'region', + 'environment', + 'runnerNamePrefix', + 'jitConfigParameterPath', + 'subnets', + 'launchTemplateName', + 'ec2instanceCriteria', + 'ec2OverrideConfig', + 'amiIdSsmParameterName', + 'tracingEnabled', + 'onDemandFailoverOnError', + 'scaleErrors', + 'useDedicatedHost', + 'ssmKmsKeyId', + 'ssmParameterTags', + ]), + 'configuration', + ); + if (!isRecord(value.ec2instanceCriteria)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2instanceCriteria'"); + } + rejectUnknownKeys( + value.ec2instanceCriteria, + new Set([ + 'instanceTypes', + 'instanceTypePriorities', + 'targetCapacityType', + 'maxSpotPrice', + 'instanceAllocationStrategy', + ]), + 'ec2instanceCriteria', + ); + + const targetCapacityType = value.ec2instanceCriteria.targetCapacityType; + if (targetCapacityType !== 'on-demand' && targetCapacityType !== 'spot') { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'targetCapacityType'"); + } + const instanceAllocationStrategy = requireString( + value.ec2instanceCriteria.instanceAllocationStrategy, + 'instanceAllocationStrategy', + /^[a-z-]+$/, + 64, + ) as RunnerInputParameters['ec2instanceCriteria']['instanceAllocationStrategy']; + const allowedAllocationStrategies = + targetCapacityType === 'spot' ? SPOT_ALLOCATION_STRATEGIES : ON_DEMAND_ALLOCATION_STRATEGIES; + if (!allowedAllocationStrategies.has(instanceAllocationStrategy)) { + throw new NonRetryableScaleSetError( + `Invalid allocation strategy '${instanceAllocationStrategy}' for '${targetCapacityType}' capacity`, + ); + } + + const jitConfigParameterPath = requireString( + value.jitConfigParameterPath, + 'jitConfigParameterPath', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ).replace(/\/$/, ''); + + return { + region: requireString(value.region, 'region', /^[a-z]{2}(?:-[a-z0-9]+)+-\d$/, 32), + environment: requireString(value.environment, 'environment', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128), + runnerNamePrefix: requirePossiblyEmptyString(value.runnerNamePrefix, 'runnerNamePrefix', /^[A-Za-z0-9._-]*$/, 45), + jitConfigParameterPath, + subnets: requireStringArray(value.subnets, 'subnets', /^subnet-[0-9a-f]+$/, 32), + launchTemplateName: requireString(value.launchTemplateName, 'launchTemplateName', /^[A-Za-z0-9()./_-]+$/, 128), + ec2instanceCriteria: { + instanceTypes: requireStringArray( + value.ec2instanceCriteria.instanceTypes, + 'instanceTypes', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ), + instanceTypePriorities: parseInstanceTypePriorities(value.ec2instanceCriteria.instanceTypePriorities), + targetCapacityType, + maxSpotPrice: optionalString( + value.ec2instanceCriteria.maxSpotPrice, + 'maxSpotPrice', + /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, + 32, + ), + instanceAllocationStrategy, + }, + ec2OverrideConfig: parseEc2OverrideConfig(value.ec2OverrideConfig), + amiIdSsmParameterName: optionalString( + value.amiIdSsmParameterName, + 'amiIdSsmParameterName', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ), + tracingEnabled: optionalBoolean(value.tracingEnabled, 'tracingEnabled'), + onDemandFailoverOnError: requireStringArray( + value.onDemandFailoverOnError ?? [], + 'onDemandFailoverOnError', + /^[A-Za-z0-9._-]+$/, + 128, + true, + ), + scaleErrors: requireStringArray(value.scaleErrors ?? [], 'scaleErrors', /^[A-Za-z0-9._-]+$/, 128, true), + useDedicatedHost: optionalBoolean(value.useDedicatedHost, 'useDedicatedHost'), + ssmKmsKeyId: optionalString(value.ssmKmsKeyId, 'ssmKmsKeyId', /^[A-Za-z0-9_:/+=,.@-]+$/, 2048), + ssmParameterTags: parseSsmTags(value.ssmParameterTags), + }; +} + +function validateFactoryInput(input: CreateEc2ScaleSetProviderInput): void { + requireString(input.runnerConfigName, 'runnerConfigName', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128); + if (!Number.isSafeInteger(input.scaleSetId) || input.scaleSetId <= 0) { + throw new NonRetryableScaleSetError('scaleSetId must be a positive safe integer'); + } + validateCanonicalGitHubScope(input.githubScope); +} + +function validateCanonicalGitHubScope(value: unknown): string { + if (typeof value !== 'string' || value.length === 0 || value.length > 2048) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + let url: URL; + try { + url = new URL(value); + } catch { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + const parts = url.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean); + if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + url.pathname = `/${parts.join('/')}`; + const canonical = url.toString().replace(/\/$/, ''); + if (canonical !== value) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + return value; +} + +function githubScopeHash(githubScope: string): string { + return createHash('sha256').update(githubScope, 'utf8').digest('hex'); +} + +function runnerIdentityFromGitHubScope(githubScope: string): { + runnerOwner: string; + runnerType: 'Org' | 'Repo'; +} { + const pathParts = new URL(githubScope).pathname.replace(/^\/+|\/+$/g, '').split('/'); + if (pathParts.length === 2 && pathParts[0].toLowerCase() !== 'enterprises') { + return { runnerOwner: pathParts.join('/'), runnerType: 'Repo' }; + } + + // The legacy EC2 tags do not have an enterprise discriminator. They remain + // informational here; exact ownership is fenced by runner config, scale-set + // ID, and the canonical GitHub-scope hash. + return { + runnerOwner: pathParts[0].toLowerCase() === 'enterprises' ? pathParts[1] : pathParts[0], + runnerType: 'Org', + }; +} + +function createClients(config: Ec2ScaleSetProviderConfig, dependencies: Ec2ScaleSetProviderDependencies) { + return { + ec2Client: dependencies.ec2Client ?? new EC2Client({ region: config.region }), + ssmClient: + dependencies.ssmClient ?? + new SSMClient({ + region: config.region, + maxAttempts: 10, + retryMode: 'adaptive', + }), + }; +} + +function safeError( + operation: ScaleSetReconcileOperation, + error: unknown, + details: Pick = {}, +): ScaleSetReconcileError { + return { + operation, + code: safeErrorCode(error), + retryable: isRetryableError(error), + ...details, + }; +} + +function safeErrorCode(error: unknown): string { + if (error instanceof NonRetryableScaleSetError) return 'INVALID_CONFIGURATION'; + if (!isRecord(error)) return 'UNEXPECTED_ERROR'; + for (const candidate of [error.name, error.code]) { + if (typeof candidate === 'string' && /^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(candidate)) { + return candidate; + } + } + return 'UNEXPECTED_ERROR'; +} + +function isRetryableError(error: unknown): boolean { + if (error instanceof NonRetryableScaleSetError) return false; + if (!isRecord(error)) return true; + + const identity = [error.name, error.code] + .filter((candidate): candidate is string => typeof candidate === 'string') + .join(' ') + .toLowerCase(); + if (/accessdenied|unauthor|forbidden|permission|validation|invalid|malformed|unsupported/.test(identity)) { + return false; + } + if (/throttl|timeout|temporar|serviceunavailable|internalserver|network|econn|socket|slowdown/.test(identity)) { + return true; + } + + const metadata = isRecord(error.$metadata) ? error.$metadata : undefined; + const status = [error.status, error.statusCode, metadata?.httpStatusCode].find( + (candidate): candidate is number => typeof candidate === 'number', + ); + if (status !== undefined) { + return status >= 500 || [408, 409, 425, 429].includes(status); + } + return true; +} + +function throwIfAborted(signal: AbortSignal, error?: unknown): void { + if (signal.aborted || (isRecord(error) && error.name === 'AbortError')) { + signal.throwIfAborted(); + throw error; + } +} + +function resultStatus( + errors: readonly ScaleSetReconcileError[], + current: number, + desired: number, + needsRunnerInventory: boolean, +) { + if (errors.some((error) => !error.retryable)) return 'non_retryable_error' as const; + if (errors.length > 0 || current < desired) return 'retryable_error' as const; + if (needsRunnerInventory) return 'retained' as const; + if (current > desired) return 'retained' as const; + return 'converged' as const; +} + +function finish(state: MutableReconcileState, desiredRunners: number): ScaleSetReconcileResult { + if (desiredRunners >= 0 && state.currentRunners < desiredRunners && state.errors.length === 0) { + state.errors.push({ + operation: 'reconcile', + code: 'CAPACITY_NOT_PROVISIONED', + retryable: true, + }); + } + return { + status: resultStatus(state.errors, state.currentRunners, desiredRunners, state.needsRunnerInventory), + desiredRunners, + currentRunners: state.currentRunners, + needsRunnerInventory: state.needsRunnerInventory, + actions: state.actions, + errors: state.errors, + }; +} + +function emptyState(currentRunners: number): MutableReconcileState { + return { + currentRunners, + needsRunnerInventory: false, + retainedUnknownResourceIds: new Set(), + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }; +} + +function retainUnknown(state: MutableReconcileState, resourceId?: string): void { + if (resourceId === undefined) { + state.actions.retainedUnknown++; + return; + } + if (state.retainedUnknownResourceIds.has(resourceId)) return; + state.retainedUnknownResourceIds.add(resourceId); + state.actions.retainedUnknown++; +} + +function validateDesiredRunners(desiredRunners: number): ScaleSetReconcileError | undefined { + if (!Number.isSafeInteger(desiredRunners) || desiredRunners < 0 || desiredRunners > 10000) { + return { + operation: 'validate', + code: 'INVALID_DESIRED_RUNNER_COUNT', + retryable: false, + }; + } + return undefined; +} + +function validateBootTimeout(bootTimeoutMinutes: number): ScaleSetReconcileError | undefined { + if ( + !Number.isSafeInteger(bootTimeoutMinutes) || + bootTimeoutMinutes < 1 || + bootTimeoutMinutes > MAX_BOOT_TIMEOUT_MINUTES + ) { + return { + operation: 'validate', + code: 'INVALID_BOOT_TIMEOUT', + retryable: false, + }; + } + return undefined; +} + +function validateInventorySignal(runnerInventoryComplete: unknown): ScaleSetReconcileError | undefined { + if (typeof runnerInventoryComplete !== 'boolean') { + return { + operation: 'validate', + code: 'INVALID_RUNNER_INVENTORY_SIGNAL', + retryable: false, + }; + } + return undefined; +} + +function ownershipTags(input: CreateEc2ScaleSetProviderInput): Tag[] { + return [ + { Key: ENVIRONMENT_TAG, Value: input.configuration.environment }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, + ]; +} + +async function listOwnedRunners( + input: CreateEc2ScaleSetProviderInput, + ec2Client: EC2Client, + signal: AbortSignal, +): Promise { + const runners: OwnedEc2Runner[] = []; + let nextToken: string | undefined; + do { + const response = await ec2Client.send( + new DescribeInstancesCommand({ + Filters: [ + { Name: 'instance-state-name', Values: ['pending', 'running'] }, + { Name: `tag:${APPLICATION_TAG}`, Values: [APPLICATION_VALUE] }, + { Name: `tag:${CREATED_BY_TAG}`, Values: [CREATED_BY_VALUE] }, + { Name: `tag:${ENVIRONMENT_TAG}`, Values: [input.configuration.environment] }, + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: [input.runnerConfigName] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: [String(input.scaleSetId)] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash(input.githubScope)] }, + ], + NextToken: nextToken, + }), + { abortSignal: signal }, + ); + nextToken = response.NextToken; + + for (const instance of response.Reservations?.flatMap((reservation) => reservation.Instances ?? []) ?? []) { + const runner = parseOwnedRunner(instance, input); + if (runner) runners.push(runner); + } + } while (nextToken); + + return runners; +} + +function parseOwnedRunner(instance: Instance, input: CreateEc2ScaleSetProviderInput): OwnedEc2Runner | undefined { + if (!instance.InstanceId) return undefined; + const tags = new Map((instance.Tags ?? []).flatMap((tag) => (tag.Key ? [[tag.Key, tag.Value]] : []))); + + if ( + tags.get(APPLICATION_TAG) !== APPLICATION_VALUE || + tags.get(CREATED_BY_TAG) !== CREATED_BY_VALUE || + tags.get(ENVIRONMENT_TAG) !== input.configuration.environment || + tags.get(EC2_RUNNER_CONFIG_TAG) !== input.runnerConfigName || + tags.get(EC2_SCALE_SET_ID_TAG) !== String(input.scaleSetId) || + tags.get(EC2_GITHUB_SCOPE_HASH_TAG) !== githubScopeHash(input.githubScope) + ) { + return undefined; + } + + const taggedRunnerId = tags.get(EC2_GITHUB_RUNNER_ID_TAG); + const githubRunnerId = taggedRunnerId === undefined ? undefined : Number(taggedRunnerId); + const rawScaleSetState = tags.get(EC2_SCALE_SET_STATE_TAG); + const scaleSetState = ['provisioning', 'publishing', 'config-published', 'retiring'].includes(rawScaleSetState ?? '') + ? (rawScaleSetState as Ec2ScaleSetState) + : undefined; + + return { + instanceId: instance.InstanceId, + launchTime: instance.LaunchTime, + githubRunnerId: Number.isSafeInteger(githubRunnerId) && githubRunnerId! > 0 ? githubRunnerId : undefined, + runnerName: tags.get(EC2_RUNNER_NAME_TAG), + scaleSetState, + }; +} + +function validRunnerState(value: ScaleSetRunnerState): boolean { + return ( + Number.isSafeInteger(value.runnerId) && + value.runnerId > 0 && + Number.isSafeInteger(value.scaleSetId) && + value.scaleSetId > 0 && + typeof value.runnerName === 'string' && + value.runnerName.length > 0 && + value.runnerName.length <= GITHUB_RUNNER_NAME_MAX_LENGTH && + ['online', 'offline', 'unknown'].includes(value.status) && + (typeof value.busy === 'boolean' || value.busy === undefined) && + ['started', 'completed', 'unknown'].includes(value.lifecycle) + ); +} + +function indexRunnerStates( + runnerStates: readonly ScaleSetRunnerState[], + scaleSetId: number, +): { byName: Map; ambiguousNames: Set; ambiguousIds: Set } { + const byName = new Map(); + const byId = new Map(); + const ambiguousNames = new Set(); + const ambiguousIds = new Set(); + + for (const state of runnerStates) { + if (!validRunnerState(state) || state.scaleSetId !== scaleSetId) continue; + if (byName.has(state.runnerName)) ambiguousNames.add(state.runnerName); + const existingName = byId.get(state.runnerId); + if (existingName !== undefined && existingName !== state.runnerName) { + ambiguousIds.add(state.runnerId); + ambiguousNames.add(existingName); + ambiguousNames.add(state.runnerName); + } + byName.set(state.runnerName, state); + byId.set(state.runnerId, state.runnerName); + } + return { byName, ambiguousNames, ambiguousIds }; +} + +function matchingRunnerState( + runner: OwnedEc2Runner, + index: ReturnType, + scaleSetId: number, +): ScaleSetRunnerState | undefined { + if (!runner.runnerName || !runner.githubRunnerId) return undefined; + if (index.ambiguousNames.has(runner.runnerName) || index.ambiguousIds.has(runner.githubRunnerId)) return undefined; + const state = index.byName.get(runner.runnerName); + if ( + !state || + state.runnerId !== runner.githubRunnerId || + state.runnerName !== runner.runnerName || + state.scaleSetId !== scaleSetId + ) { + return undefined; + } + return state; +} + +function isWithinBootTimeout(runner: OwnedEc2Runner, bootTimeoutMinutes: number, now: number): boolean { + const launchTime = runner.launchTime?.getTime(); + if (launchTime === undefined || !Number.isFinite(launchTime)) return false; + const ageMilliseconds = now - launchTime; + return ageMilliseconds >= 0 && ageMilliseconds < bootTimeoutMinutes * 60_000; +} + +function isConfirmedServingState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.status === 'online'; +} + +function servingCapacity( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + now: number, +): OwnedEc2Runner[] { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const serving: OwnedEc2Runner[] = []; + + for (const runner of runners) { + if (runner.scaleSetState !== 'config-published') { + // An interrupted publication may already have been consumed. Preserve it, + // but do not let it suppress replacement capacity indefinitely. + retainUnknown(state, runner.instanceId); + continue; + } + + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + if (githubState !== undefined && isConfirmedServingState(githubState)) { + serving.push(runner); + continue; + } + if (isWithinBootTimeout(runner, request.bootTimeoutMinutes, now)) { + serving.push(runner); + continue; + } + + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) { + // Treat the stale handoff provisionally as serving until the controller + // supplies one complete joined inventory. This avoids a blind replacement + // before GitHub identity can be checked. + state.needsRunnerInventory = true; + serving.push(runner); + } + } + + return serving; +} + +function isSafeScaleDownState(state: ScaleSetRunnerState): boolean { + return ( + (state.lifecycle === 'completed' && state.busy !== true) || + (state.lifecycle !== 'started' && state.status === 'online' && state.busy === false) + ); +} + +function isBusyState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.busy === true; +} + +async function tagRunner(instanceId: string, tags: Tag[], ec2Client: EC2Client, signal: AbortSignal): Promise { + await ec2Client.send(new CreateTagsCommand({ Resources: [instanceId], Tags: tags }), { abortSignal: signal }); +} + +async function getParameter(name: string, ssmClient: SSMClient, signal: AbortSignal): Promise { + const response = await ssmClient.send(new GetParameterCommand({ Name: name, WithDecryption: true }), { + abortSignal: signal, + }); + if (!response.Parameter?.Value) { + throw new NonRetryableScaleSetError(`AMI parameter '${name}' has no value`); + } + return response.Parameter.Value; +} + +function jitParameterName(config: Ec2ScaleSetProviderConfig, instanceId: string): string { + return `${config.jitConfigParameterPath}/${instanceId}`; +} + +function jitParameterTags(input: CreateEc2ScaleSetProviderInput, instanceId: string): SsmTag[] { + const reserved = new Set(['InstanceId', EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG, EC2_GITHUB_SCOPE_HASH_TAG]); + return [ + ...(input.configuration.ssmParameterTags ?? []).filter((tag) => tag.Key && !reserved.has(tag.Key)), + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + ]; +} + +async function publishJitConfiguration( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + encodedJitConfiguration: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + const valueSize = Buffer.byteLength(encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new NonRetryableScaleSetError('JIT configuration must be between 1 and 8192 bytes'); + } + + await ssmClient.send( + new PutParameterCommand({ + Name: jitParameterName(input.configuration, instanceId), + Value: encodedJitConfiguration, + Type: 'SecureString', + KeyId: input.configuration.ssmKmsKeyId, + Overwrite: false, + Tier: valueSize >= SSM_STANDARD_TIER_THRESHOLD ? 'Advanced' : 'Standard', + Tags: jitParameterTags(input, instanceId), + }), + { abortSignal: signal }, + ); +} + +async function bestEffortCancelJitPublication( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + try { + await ssmClient.send(new DeleteParameterCommand({ Name: jitParameterName(input.configuration, instanceId) }), { + abortSignal: signal, + }); + } catch (error) { + throwIfAborted(signal, error); + } +} + +function validateJitResult( + result: GenerateScaleSetJitConfigurationResult, + expectedRunnerName: string, + scaleSetId: number, +): void { + if ( + !Number.isSafeInteger(result.runnerId) || + result.runnerId <= 0 || + result.runnerName !== expectedRunnerName || + result.scaleSetId !== scaleSetId + ) { + throw new NonRetryableScaleSetError('JIT configuration returned an unexpected runner identity'); + } + const valueSize = Buffer.byteLength(result.encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new NonRetryableScaleSetError('JIT configuration has an invalid size'); + } +} + +async function terminateUnpublishedRunner( + instanceId: string, + state: MutableReconcileState, + ec2Client: EC2Client, + signal: AbortSignal, +): Promise { + try { + await terminateRunner(instanceId, { ec2Client, signal }); + state.currentRunners--; + state.actions.terminated++; + } catch (error) { + throwIfAborted(signal, error); + retainUnknown(state, instanceId); + state.errors.push(safeError('terminate', error, { resourceId: instanceId })); + } +} + +async function cleanGitHubRunner( + jit: GenerateScaleSetJitConfigurationResult, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, +): Promise { + try { + await request.removeRunner({ + runnerId: jit.runnerId, + runnerName: jit.runnerName, + scaleSetId: jit.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('remove_runner', error, { runnerName: jit.runnerName })); + } +} + +async function configureLaunchedRunner( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + clients: ReturnType, +): Promise { + const runnerName = `${input.configuration.runnerNamePrefix}${instanceId}`; + if (runnerName.length > GITHUB_RUNNER_NAME_MAX_LENGTH) { + state.errors.push({ + operation: 'generate_jit_configuration', + code: 'RUNNER_NAME_TOO_LONG', + retryable: false, + resourceId: instanceId, + }); + await terminateUnpublishedRunner(instanceId, state, clients.ec2Client, request.signal); + return; + } + + let jit: GenerateScaleSetJitConfigurationResult; + try { + jit = await request.generateJitConfiguration({ runnerName, signal: request.signal }); + validateJitResult(jit, runnerName, input.scaleSetId); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('generate_jit_configuration', error, { runnerName, resourceId: instanceId })); + await terminateUnpublishedRunner(instanceId, state, clients.ec2Client, request.signal); + return; + } + + try { + await tagRunner( + instanceId, + [ + { Key: EC2_RUNNER_NAME_TAG, Value: jit.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(jit.runnerId) }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, + ], + clients.ec2Client, + request.signal, + ); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + await cleanGitHubRunner(jit, request, state); + await terminateUnpublishedRunner(instanceId, state, clients.ec2Client, request.signal); + return; + } + + try { + await publishJitConfiguration(input, instanceId, jit.encodedJitConfiguration, clients.ssmClient, request.signal); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('publish_jit_configuration', error, { runnerName, resourceId: instanceId })); + await bestEffortCancelJitPublication(input, instanceId, clients.ssmClient, request.signal); + // Main's bootstrap reads before deleting. Even a successful controller-side + // DeleteParameter can race after that read and cannot prove non-consumption. + // Preserve both GitHub and compute state until an exact lifecycle signal is observed. + retainUnknown(state, instanceId); + return; + } + + state.actions.launched++; + try { + await tagRunner( + instanceId, + [{ Key: EC2_SCALE_SET_STATE_TAG, Value: 'config-published' }], + clients.ec2Client, + request.signal, + ); + } catch (error) { + throwIfAborted(request.signal, error); + // Publication may already have been consumed. Preserve the instance and exact GitHub identity. + retainUnknown(state, instanceId); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + } +} + +async function scaleUp( + input: CreateEc2ScaleSetProviderInput, + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + clients: ReturnType, +): Promise { + const runnerIdentity = runnerIdentityFromGitHubScope(input.githubScope); + let createResult; + try { + createResult = await createRunner( + { + environment: input.configuration.environment, + runnerOwner: runnerIdentity.runnerOwner, + runnerType: runnerIdentity.runnerType, + subnets: input.configuration.subnets, + launchTemplateName: input.configuration.launchTemplateName, + ec2instanceCriteria: input.configuration.ec2instanceCriteria, + ec2OverrideConfig: input.configuration.ec2OverrideConfig, + numberOfRunners: count, + source: CREATED_BY_VALUE, + amiIdSsmParameterName: input.configuration.amiIdSsmParameterName, + tracingEnabled: input.configuration.tracingEnabled, + onDemandFailoverOnError: input.configuration.onDemandFailoverOnError, + scaleErrors: input.configuration.scaleErrors, + useDedicatedHost: input.configuration.useDedicatedHost, + additionalTags: ownershipTags(input), + }, + { + ec2Client: clients.ec2Client, + getParameter: (name) => getParameter(name, clients.ssmClient, request.signal), + signal: request.signal, + }, + ); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error)); + return; + } + + state.currentRunners += createResult.instances.length; + if (createResult.retryableErrorCount > 0) { + state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_RETRYABLE', retryable: true }); + } + if (createResult.nonRetryableErrorCount > 0) { + state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_NON_RETRYABLE', retryable: false }); + } + + for (const instanceId of createResult.instances) { + request.signal.throwIfAborted(); + await configureLaunchedRunner(input, instanceId, request, state, clients); + } +} + +async function terminateKnownIdleRunner( + runner: OwnedEc2Runner, + githubState: ScaleSetRunnerState, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + clients: ReturnType, +): Promise { + let removalResult; + try { + removalResult = await request.removeRunner({ + runnerId: githubState.runnerId, + runnerName: githubState.runnerName, + scaleSetId: githubState.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('remove_runner', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } + + if (removalResult.status === 'retained_busy') { + state.actions.retainedBusy++; + return false; + } + if (removalResult.status !== 'removed') { + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; + return false; + } + + try { + await terminateRunner(runner.instanceId, { ec2Client: clients.ec2Client, signal: request.signal }); + state.currentRunners--; + state.actions.terminated++; + return true; + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('terminate', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } +} + +async function scaleDown( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + clients: ReturnType, +): Promise { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const candidates: { runner: OwnedEc2Runner; githubState: ScaleSetRunnerState }[] = []; + + for (const runner of runners) { + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + if (!githubState) { + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; + } else if (isBusyState(githubState)) { + state.actions.retainedBusy++; + } else if (isSafeScaleDownState(githubState)) { + candidates.push({ runner, githubState }); + } else { + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; + } + } + + candidates.sort((left, right) => { + const launchOrder = (right.runner.launchTime?.getTime() ?? 0) - (left.runner.launchTime?.getTime() ?? 0); + return launchOrder || left.runner.instanceId.localeCompare(right.runner.instanceId); + }); + + let remaining = count; + for (const candidate of candidates) { + if (remaining === 0) break; + request.signal.throwIfAborted(); + if (await terminateKnownIdleRunner(candidate.runner, candidate.githubState, request, state, clients)) { + remaining--; + } + } +} + +export function createEc2ScaleSetProvider( + input: CreateEc2ScaleSetProviderInput, + dependencies: Ec2ScaleSetProviderDependencies = {}, +): ScaleSetComputeProvider { + const normalizedInput = { + ...input, + configuration: parseEc2ScaleSetProviderConfig(input.configuration), + }; + validateFactoryInput(normalizedInput); + const clients = createClients(normalizedInput.configuration, dependencies); + const now = dependencies.now ?? Date.now; + + return { + async reconcile(request): Promise { + request.signal.throwIfAborted(); + const validationError = + validateDesiredRunners(request.desiredRunners) ?? + validateBootTimeout(request.bootTimeoutMinutes) ?? + validateInventorySignal(request.runnerInventoryComplete); + if (validationError) { + const state = emptyState(0); + state.errors.push(validationError); + return finish(state, request.desiredRunners); + } + + let runners: OwnedEc2Runner[]; + try { + runners = await listOwnedRunners(normalizedInput, clients.ec2Client, request.signal); + } catch (error) { + throwIfAborted(request.signal, error); + const state = emptyState(0); + state.errors.push(safeError('list', error)); + return finish(state, request.desiredRunners); + } + + const state = emptyState(runners.length); + const servingRunners = servingCapacity(normalizedInput, runners, request, state, now()); + + if (servingRunners.length < request.desiredRunners) { + const capacityDeficit = request.desiredRunners - servingRunners.length; + const availableReplacementSlots = Math.max( + 0, + request.desiredRunners + RETAINED_CAPACITY_REPLACEMENT_SURGE - runners.length, + ); + const launchCount = Math.min(capacityDeficit, availableReplacementSlots); + if (launchCount > 0) { + await scaleUp(normalizedInput, launchCount, request, state, clients); + } + } else if (servingRunners.length > request.desiredRunners) { + await scaleDown( + normalizedInput, + servingRunners, + servingRunners.length - request.desiredRunners, + request, + state, + clients, + ); + } + + return finish(state, request.desiredRunners); + }, + }; +} diff --git a/lambdas/libs/compute-providers/core/index.ts b/lambdas/libs/compute-providers/core/index.ts index c5560942fa..ece1bc603e 100644 --- a/lambdas/libs/compute-providers/core/index.ts +++ b/lambdas/libs/compute-providers/core/index.ts @@ -6,7 +6,7 @@ export interface ComputeProvider { type: ComputeProviderType; } -export type RunnerSource = 'scale-up-lambda' | 'pool-lambda'; +export type RunnerSource = 'scale-up-lambda' | 'pool-lambda' | 'scale-set-service'; export type RunnerType = 'Org' | 'Repo'; export interface CreateGitHubRunnerConfig { diff --git a/lambdas/libs/compute-providers/package.json b/lambdas/libs/compute-providers/package.json index c1806818cc..0ab5bb25de 100644 --- a/lambdas/libs/compute-providers/package.json +++ b/lambdas/libs/compute-providers/package.json @@ -8,8 +8,10 @@ "./provider-types": "./provider-types.ts", "./webhook": "./webhook.ts", "./control-plane": "./control-plane.ts", + "./scale-set": "./scale-set.ts", "./aws/ec2/webhook": "./aws/ec2/webhook.ts", "./aws/ec2/control-plane": "./aws/ec2/control-plane.ts", + "./aws/ec2/scale-set": "./aws/ec2/scale-set.ts", "./aws/ec2/runners": "./aws/ec2/src/runners.ts", "./aws/ec2/control-plane/runner-creation": "./aws/ec2/src/control-plane/runner-creation.ts" }, @@ -27,6 +29,7 @@ "@aws-github-runner/aws-powertools-util": "*", "@aws-github-runner/aws-ssm-util": "*", "@aws-sdk/client-ec2": "^3.1009.0", + "@aws-sdk/client-ssm": "^3.1009.0", "@octokit/rest": "22.0.1", "moment": "2.29.4", "yn": "3.1.1" diff --git a/lambdas/libs/compute-providers/providers.config.scale-set.ts b/lambdas/libs/compute-providers/providers.config.scale-set.ts new file mode 100644 index 0000000000..7f8f0ecc64 --- /dev/null +++ b/lambdas/libs/compute-providers/providers.config.scale-set.ts @@ -0,0 +1,5 @@ +import { provider as ec2 } from './aws/ec2/scale-set'; +import type { ScaleSetComputeProviderModule } from './scale-set'; + +/** Provider plugins included in the scale-set service bundle. */ +export const enabledScaleSetProviders = [ec2] as const satisfies readonly ScaleSetComputeProviderModule[]; diff --git a/lambdas/libs/compute-providers/scale-set.test.ts b/lambdas/libs/compute-providers/scale-set.test.ts new file mode 100644 index 0000000000..e3cd187ce6 --- /dev/null +++ b/lambdas/libs/compute-providers/scale-set.test.ts @@ -0,0 +1,125 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { createEc2ScaleSetPlugin } from './aws/ec2/scale-set'; +import { + createScaleSetComputeProviderRegistry, + type ScaleSetComputeProviderPlugin, + validateScaleSetProviderEnvironmentVariables, +} from './scale-set'; + +const configuration = { + region: 'eu-west-1', + environment: 'unit-test', + runnerNamePrefix: 'runner-', + jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', + subnets: ['subnet-12345678'], + launchTemplateName: 'unit-test-runners', + ec2instanceCriteria: { + instanceTypes: ['m7i.large'], + targetCapacityType: 'on-demand', + instanceAllocationStrategy: 'lowest-price', + }, + scaleErrors: [], +}; + +describe('scale-set compute-provider registry', () => { + it('creates a separate provider instance for every runner config', () => { + const registry = createScaleSetComputeProviderRegistry([createEc2ScaleSetPlugin()]); + const first = registry.create('ec2', { + runnerConfigName: 'shared', + scaleSetId: 1, + githubScope: 'https://github.com/first', + configuration, + }); + const second = registry.create('ec2', { + runnerConfigName: 'shared', + scaleSetId: 1, + githubScope: 'https://github.com/second', + configuration, + }); + + expect(first).not.toBe(second); + expect(first.reconcile).toEqual(expect.any(Function)); + expect(second.reconcile).toEqual(expect.any(Function)); + expect(registry.environmentVariables('ec2')).toEqual({}); + expect(Object.isFrozen(registry.environmentVariables('ec2'))).toBe(true); + }); + + it('rejects duplicate and missing plugins explicitly', () => { + const plugin: ScaleSetComputeProviderPlugin = { + type: 'test', + capabilities: { + environmentVariables: {}, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }; + + expect(() => createScaleSetComputeProviderRegistry([plugin, plugin])).toThrow( + "Duplicate scale-set compute provider plugin 'test'", + ); + expect(() => + createScaleSetComputeProviderRegistry([]).create('missing', { + runnerConfigName: 'runner', + scaleSetId: 1, + githubScope: 'https://github.com/example', + configuration: {}, + }), + ).toThrow("No scale-set compute provider plugin registered for 'missing'"); + expect(() => createScaleSetComputeProviderRegistry([]).environmentVariables('missing')).toThrow( + "No scale-set compute provider plugin registered for 'missing'", + ); + }); + + it('returns a validated immutable provider environment', () => { + const source = { EC2_ENDPOINT_MODE: 'regional' }; + const registry = createScaleSetComputeProviderRegistry([ + { + type: 'test', + capabilities: { + environmentVariables: source, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }, + ]); + + const environment = registry.environmentVariables('test'); + source.EC2_ENDPOINT_MODE = 'changed-after-registration'; + expect(environment).toEqual({ EC2_ENDPOINT_MODE: 'regional' }); + expect(Object.isFrozen(environment)).toBe(true); + }); + + it.each>>([ + { AWS_REGION: 'eu-west-1' }, + { SCALE_SET_OVERRIDE: 'unsafe' }, + { NODE_OPTIONS: '--import=untrusted' }, + { PATH: '/untrusted' }, + { lower_case: 'value' }, + { VALID_NAME: 'line\nbreak' }, + { VALID_NAME: 'x'.repeat(4097) }, + ])('rejects reserved or unsafe provider environment variables: %o', (environmentVariables) => { + expect(() => + createScaleSetComputeProviderRegistry([ + { + type: 'test', + capabilities: { + environmentVariables, + create: vi.fn(() => ({ reconcile: vi.fn() })), + }, + }, + ]), + ).toThrow(/reserved or invalid|invalid value/); + }); + + it('rejects malformed or oversized provider environments', () => { + expect(() => validateScaleSetProviderEnvironmentVariables(null as never)).toThrow('must be an object'); + expect(() => validateScaleSetProviderEnvironmentVariables([] as never)).toThrow('must be an object'); + expect(() => validateScaleSetProviderEnvironmentVariables({ VALID_NAME: 42 } as never)).toThrow( + 'has an invalid value', + ); + expect(() => + validateScaleSetProviderEnvironmentVariables( + Object.fromEntries(Array.from({ length: 65 }, (_, index) => [`PROVIDER_${index}`, 'value'])), + ), + ).toThrow('must contain at most 64 entries'); + }); +}); diff --git a/lambdas/libs/compute-providers/scale-set.ts b/lambdas/libs/compute-providers/scale-set.ts new file mode 100644 index 0000000000..6f06a91c08 --- /dev/null +++ b/lambdas/libs/compute-providers/scale-set.ts @@ -0,0 +1,209 @@ +import { enabledScaleSetProviders } from './providers.config.scale-set'; + +export type ScaleSetRunnerStatus = 'online' | 'offline' | 'unknown'; +export type ScaleSetRunnerLifecycle = 'started' | 'completed' | 'unknown'; + +/** + * Controller-observed GitHub state for one runner. + * + * The compute provider treats missing, duplicate, or unrecognized state as + * unknown. Callers must not infer `busy: false` when GitHub did not provide a + * busy state. + */ +export interface ScaleSetRunnerState { + runnerId: number; + runnerName: string; + scaleSetId: number; + status: ScaleSetRunnerStatus; + busy: boolean | undefined; + lifecycle: ScaleSetRunnerLifecycle; +} + +export interface GenerateScaleSetJitConfigurationInput { + runnerName: string; + signal?: AbortSignal; +} + +export interface GenerateScaleSetJitConfigurationResult { + encodedJitConfiguration: string; + runnerId: number; + runnerName: string; + scaleSetId: number; +} + +export type GenerateScaleSetJitConfiguration = ( + input: GenerateScaleSetJitConfigurationInput, +) => Promise; + +export interface RemoveScaleSetRunnerInput { + runnerId: number; + runnerName: string; + scaleSetId: number; + signal?: AbortSignal; +} + +export type ScaleSetRemoveRunnerStatus = 'removed' | 'retained_busy' | 'retained_unknown'; + +export interface ScaleSetRemoveRunnerResult { + status: ScaleSetRemoveRunnerStatus; +} + +export type RemoveScaleSetRunner = (input: RemoveScaleSetRunnerInput) => Promise; + +export interface ScaleSetReconcileRequest { + desiredRunners: number; + /** Orchestration-owned handoff window before exact runner inventory is required. */ + bootTimeoutMinutes: number; + /** True only when runnerStates contains the controller's complete, freshly joined Actions and GitHub inventory. */ + runnerInventoryComplete: boolean; + runnerStates: readonly ScaleSetRunnerState[]; + signal: AbortSignal; + generateJitConfiguration: GenerateScaleSetJitConfiguration; + removeRunner: RemoveScaleSetRunner; +} + +export type ScaleSetReconcileStatus = 'converged' | 'retained' | 'retryable_error' | 'non_retryable_error'; + +export type ScaleSetReconcileOperation = + | 'validate' + | 'reconcile' + | 'list' + | 'launch' + | 'generate_jit_configuration' + | 'publish_jit_configuration' + | 'remove_runner' + | 'terminate'; + +/** Error metadata is deliberately bounded and never contains a JIT configuration or raw upstream error message. */ +export interface ScaleSetReconcileError { + operation: ScaleSetReconcileOperation; + code: string; + retryable: boolean; + runnerName?: string; + resourceId?: string; +} + +export interface ScaleSetReconcileActions { + launched: number; + terminated: number; + retainedBusy: number; + retainedUnknown: number; +} + +export interface ScaleSetReconcileResult { + status: ScaleSetReconcileStatus; + desiredRunners: number; + /** Best-known owned capacity after actions completed; the next reconciliation re-observes AWS. */ + currentRunners: number; + /** The provider retained unknown capacity and needs a controller inventory refresh before retrying scale-down. */ + needsRunnerInventory: boolean; + actions: ScaleSetReconcileActions; + errors: readonly ScaleSetReconcileError[]; +} + +export interface ScaleSetComputeProvider { + reconcile(request: ScaleSetReconcileRequest): Promise; +} + +export interface ScaleSetComputeProviderFactoryInput { + runnerConfigName: string; + scaleSetId: number; + /** Canonical GitHub configuration URL used as an immutable provider ownership scope. */ + githubScope: string; + /** Provider-owned configuration. The selected provider validates it before use. */ + configuration: unknown; +} + +export interface ScaleSetComputeProviderCapabilities { + /** Provider-owned, non-secret task environment. Values are validated and immutable after registration. */ + environmentVariables: Readonly>; + create(input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider; +} + +export interface ScaleSetComputeProviderPlugin { + type: TType; + capabilities: ScaleSetComputeProviderCapabilities; +} + +export interface ScaleSetComputeProviderModule { + type: TType; + createPlugin(): ScaleSetComputeProviderPlugin; +} + +const SCALE_SET_ENVIRONMENT_KEY = /^[A-Z][A-Z0-9_]{0,127}$/; +const RESERVED_SCALE_SET_ENVIRONMENT_PREFIXES = ['AWS_', 'ECS_', 'GITHUB_', 'SCALE_SET_', 'NODE_']; +const RESERVED_SCALE_SET_ENVIRONMENT_KEYS = new Set(['PATH', 'HOME', 'HOSTNAME', 'PWD', 'SHLVL']); +const MAX_SCALE_SET_ENVIRONMENT_VARIABLES = 64; +const MAX_SCALE_SET_ENVIRONMENT_VALUE_BYTES = 4096; + +export function validateScaleSetProviderEnvironmentVariables( + value: Readonly>, +): Readonly> { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new Error('Scale-set provider environmentVariables must be an object'); + } + const entries = Object.entries(value); + if (entries.length > MAX_SCALE_SET_ENVIRONMENT_VARIABLES) { + throw new Error( + `Scale-set provider environmentVariables must contain at most ${MAX_SCALE_SET_ENVIRONMENT_VARIABLES} entries`, + ); + } + + const normalized = Object.create(null) as Record; + for (const [key, environmentValue] of entries) { + if ( + !SCALE_SET_ENVIRONMENT_KEY.test(key) || + RESERVED_SCALE_SET_ENVIRONMENT_KEYS.has(key) || + RESERVED_SCALE_SET_ENVIRONMENT_PREFIXES.some((prefix) => key.startsWith(prefix)) + ) { + throw new Error(`Scale-set provider environment variable '${key}' is reserved or invalid`); + } + if ( + typeof environmentValue !== 'string' || + Buffer.byteLength(environmentValue, 'utf8') > MAX_SCALE_SET_ENVIRONMENT_VALUE_BYTES || + [...environmentValue].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }) + ) { + throw new Error(`Scale-set provider environment variable '${key}' has an invalid value`); + } + normalized[key] = environmentValue; + } + return Object.freeze(normalized); +} + +export function createScaleSetComputeProviderRegistry( + plugins: readonly ScaleSetComputeProviderPlugin[] = enabledScaleSetProviders.map((provider) => + provider.createPlugin(), + ), +) { + const pluginsByType = new Map(); + const environmentVariablesByType = new Map>>(); + for (const plugin of plugins) { + if (pluginsByType.has(plugin.type)) { + throw new Error(`Duplicate scale-set compute provider plugin '${plugin.type}'`); + } + pluginsByType.set(plugin.type, plugin); + environmentVariablesByType.set( + plugin.type, + validateScaleSetProviderEnvironmentVariables(plugin.capabilities.environmentVariables), + ); + } + + function get(type: string): ScaleSetComputeProviderPlugin { + const plugin = pluginsByType.get(type); + if (!plugin) throw new Error(`No scale-set compute provider plugin registered for '${type}'`); + return plugin; + } + + return { + create(type: string, input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider { + return get(type).capabilities.create(input); + }, + environmentVariables(type: string): Readonly> { + get(type); + return environmentVariablesByType.get(type)!; + }, + }; +} diff --git a/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset b/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset new file mode 100644 index 0000000000..28a50fa226 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/LICENSE.actions-scaleset @@ -0,0 +1,21 @@ +MIT License + +Copyright GitHub, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lambdas/libs/github-actions-scale-set/README.md b/lambdas/libs/github-actions-scale-set/README.md new file mode 100644 index 0000000000..3892c678a2 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/README.md @@ -0,0 +1,69 @@ +# GitHub Actions runner scale-set client for TypeScript + +This workspace package implements the GitHub Actions runner scale-set protocol with native `fetch`. It is intended for the `terraform-aws-github-runner` control plane and can also be reused by other Node.js scale-set listeners. + +The protocol is currently a GitHub Public Preview. This implementation follows the public [`actions/scaleset`](https://github.com/actions/scaleset) Go client at commit [`cb0405b`](https://github.com/actions/scaleset/tree/cb0405b2d874500e75ae34eff8d582ab75956b45). + +The upstream copyright and MIT permission notice are retained in [`LICENSE.actions-scaleset`](./LICENSE.actions-scaleset). + +## What it provides + +- HTTPS organization, repository, enterprise, GitHub.com, and GHES registration URLs; +- PAT authentication or an asynchronous access-token provider for existing GitHub App authentication; +- runner scale-set CRUD and runner-group lookup; +- just-in-time runner configuration generation; +- runner lookup and removal; +- message-session creation, refresh, long polling, acknowledgement, and job acquisition; +- bounded retries for idempotent requests that encounter network failures, HTTP 429, or HTTP 5xx responses. + +There is no scale-up or scale-down REST operation. A listener polls the message queue and reports its maximum capacity. GitHub returns `statistics.totalAssignedJobs`; the caller reconciles its compute capacity to that value and terminates ephemeral compute after `JobCompleted` messages. + +## Basic usage + +```ts +import { GitHubActionsScaleSetClient } from '@aws-github-runner/github-actions-scale-set'; + +const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + accessTokenProvider: async () => installationAccessToken, + systemInfo: { + system: 'terraform-aws-github-runner', + subsystem: 'scale-set-listener', + }, + retry: { + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 5 * 60_000, + }, +}); + +const scaleSet = await client.createRunnerScaleSet({ + name: 'linux-x64', + runnerGroupId: 1, + runnerSetting: { disableUpdate: true }, +}); + +const session = await client.createMessageSessionClient(scaleSet.id!, 'listener-01'); + +try { + const message = await session.getMessage(0, 20); + if (message) { + const availableIds = message.jobAvailableMessages.map((job) => job.runnerRequestId); + await session.acquireJobs(availableIds); + + // Reconcile compute from message.statistics.totalAssignedJobs and use + // client.generateJitRunnerConfig(...) for every runner being created. + + await session.deleteMessage(message.messageId); + } +} finally { + await session.close(); +} +``` + +Treat encoded JIT configurations and all access tokens as secrets. A message should be acknowledged only after its compute and completion handling succeeds so it can be redelivered after a failure. + +The retry values shown above are the defaults. Automatic transport retries apply only to idempotent methods (`GET`, `HEAD`, `OPTIONS`, `PUT`, and `DELETE`). Non-idempotent `POST` and `PATCH` operations, including JIT generation, session creation, and job acquisition, are attempted once so a lost response cannot cause the operation to be replayed. `Retry-After` is honored for eligible 429/5xx responses and capped by `maxBackoffMs`. Caller cancellation interrupts both an active request and retry backoff. + +The `/actions/runner-registration` admin bootstrap is the sole narrowly scoped exception: its `POST` retries transient transport failures and 429/5xx responses, plus 401/403 while RemoteAuth propagates. Queue 401 responses are not transport-retried; they trigger the message-session token refresh flow once. diff --git a/lambdas/libs/github-actions-scale-set/package.json b/lambdas/libs/github-actions-scale-set/package.json new file mode 100644 index 0000000000..8e0d87576a --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/package.json @@ -0,0 +1,35 @@ +{ + "name": "@aws-github-runner/github-actions-scale-set", + "version": "1.0.0", + "main": "src/index.ts", + "type": "module", + "license": "MIT", + "exports": { + ".": "./src/index.ts", + "./config": "./src/config.ts", + "./errors": "./src/errors.ts", + "./types": "./src/types.ts" + }, + "scripts": { + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "typecheck": "tsc --noEmit", + "all": "yarn format && yarn lint && yarn typecheck && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "typescript": "^5.9.3" + }, + "nx": { + "includedScripts": [ + "format", + "format-check", + "lint", + "typecheck", + "all" + ] + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/client.test.ts b/lambdas/libs/github-actions-scale-set/src/client.test.ts new file mode 100644 index 0000000000..3a14a7591d --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/client.test.ts @@ -0,0 +1,437 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { GitHubActionsScaleSetClient } from './client'; +import { SCALE_SET_ERROR_CODES, ScaleSetHttpError, ScaleSetProtocolError } from './errors'; +import { ScaleSetFetch } from './types'; + +type RequestInput = Parameters[0]; +type ServiceHandler = (url: URL, init: RequestInit) => Response | Promise; + +function requestUrl(input: RequestInput): URL { + if (input instanceof Request) { + return new URL(input.url); + } + return new URL(input.toString()); +} + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function actionsAdminToken(expiresAt = Math.floor(Date.now() / 1000) + 60 * 60): string { + const header = Buffer.from(JSON.stringify({ alg: 'RS256', typ: 'JWT' })).toString('base64url'); + const payload = Buffer.from(JSON.stringify({ exp: expiresAt })).toString('base64url'); + return `${header}.${payload}.signature`; +} + +function clientFixture(serviceHandler: ServiceHandler, options: { adminToken?: () => string; now?: () => Date } = {}) { + const accessTokenProvider = vi.fn(async () => 'github-access-token'); + const registrationRequests: Array<{ url: URL; init: RequestInit }> = []; + const serviceRequests: Array<{ url: URL; init: RequestInit }> = []; + const fetchImplementation = vi.fn(async (input, init = {}) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + registrationRequests.push({ url, init }); + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + registrationRequests.push({ url, init }); + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: options.adminToken?.() ?? actionsAdminToken(), + }); + } + + serviceRequests.push({ url, init }); + return serviceHandler(url, init); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + accessTokenProvider, + fetch: fetchImplementation, + systemInfo: { system: 'unit-test', subsystem: 'sdk' }, + now: options.now, + }); + + return { + accessTokenProvider, + client, + fetchImplementation, + registrationRequests, + serviceRequests, + }; +} + +describe('GitHubActionsScaleSetClient', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('bootstraps Actions authentication once and reuses the unexpired admin token', async () => { + const fixture = clientFixture(() => jsonResponse({ count: 0, value: [] })); + + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + + expect(fixture.accessTokenProvider).toHaveBeenCalledOnce(); + expect(fixture.registrationRequests).toHaveLength(2); + expect(fixture.serviceRequests).toHaveLength(2); + + const registrationTokenRequest = fixture.registrationRequests[0]; + expect(registrationTokenRequest.url.toString()).toBe( + 'https://api.github.com/orgs/example/actions/runners/registration-token', + ); + expect(new Headers(registrationTokenRequest.init.headers).get('Authorization')).toBe('Bearer github-access-token'); + expect(new Headers(registrationTokenRequest.init.headers).get('Content-Type')).toBe( + 'application/vnd.github.v3+json', + ); + expect(JSON.parse(new Headers(registrationTokenRequest.init.headers).get('User-Agent') as string)).toMatchObject({ + build_commit_sha: '', + kind: 'scaleset', + system: 'unit-test', + }); + + const adminConnectionRequest = fixture.registrationRequests[1]; + expect(adminConnectionRequest.url.toString()).toBe('https://api.github.com/actions/runner-registration'); + expect(new Headers(adminConnectionRequest.init.headers).get('Authorization')).toBe( + 'RemoteAuth runner-registration-token', + ); + expect(JSON.parse(adminConnectionRequest.init.body as string)).toEqual({ + url: 'https://github.com/example', + runner_event: 'register', + }); + + for (const { url, init } of fixture.serviceRequests) { + expect(url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets'); + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + expect(url.searchParams.get('runnerGroupId')).toBe('4'); + expect(url.searchParams.get('name')).toBe('linux'); + expect(new Headers(init.headers).get('Authorization')).toMatch(/^Bearer /); + } + }); + + it.each([ + 'http://actions.example/tenant/123', + 'https://user:password@actions.example/tenant/123', + 'https://actions.example/tenant/123?signature=secret', + 'https://actions.example/tenant/123#fragment', + ])('rejects an unsafe Actions service admin URL: %s', async (unsafeUrl) => { + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ url: unsafeUrl, token: actionsAdminToken() }); + } + return new Response(null, { status: 500 }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + }); + + await expect(client.getRunnerScaleSetById(42)).rejects.toBeInstanceOf(ScaleSetProtocolError); + }); + + it('fetches an exact public GitHub runner immediately by id', async () => { + const fixture = clientFixture((url) => { + if (url.pathname === '/orgs/example/actions/runners/71') { + return jsonResponse({ id: 71, name: 'runner-71', status: 'online', busy: false }); + } + return new Response(null, { status: 500 }); + }); + + await expect(fixture.client.getGitHubRunner(71)).resolves.toEqual({ + id: 71, + name: 'runner-71', + status: 'online', + busy: false, + }); + expect(fixture.accessTokenProvider).toHaveBeenCalledOnce(); + }); + + it('refreshes the Actions admin token when it enters the 60-second expiry window', async () => { + let nowMs = Date.UTC(2026, 7, 14, 12, 0, 0); + let tokenIssue = 0; + const fixture = clientFixture(() => jsonResponse({ count: 0, value: [] }), { + now: () => new Date(nowMs), + adminToken: () => { + tokenIssue += 1; + const lifetimeSeconds = tokenIssue === 1 ? 120 : 3_600; + return actionsAdminToken(Math.floor(nowMs / 1000) + lifetimeSeconds); + }, + }); + + await fixture.client.getRunnerScaleSet(4, 'linux'); + nowMs += 70_000; + await fixture.client.getRunnerScaleSet(4, 'linux'); + + expect(fixture.accessTokenProvider).toHaveBeenCalledTimes(2); + expect(fixture.registrationRequests).toHaveLength(4); + expect(tokenIssue).toBe(2); + }); + + it('retries transient and propagation failures only while bootstrapping the admin connection', async () => { + let registrationTokenRequests = 0; + let adminConnectionRequests = 0; + let actionsRequests = 0; + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + registrationTokenRequests += 1; + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + adminConnectionRequests += 1; + if (adminConnectionRequests === 1) { + return jsonResponse({ message: 'temporarily unavailable' }, 503); + } + if (adminConnectionRequests === 2) { + return jsonResponse({ message: 'not propagated' }, 401); + } + if (adminConnectionRequests === 3) { + return jsonResponse({ message: 'not propagated' }, 403); + } + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + + actionsRequests += 1; + return jsonResponse({ count: 0, value: [] }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + retry: { + maxRetries: 3, + initialBackoffMs: 0, + maxBackoffMs: 0, + requestTimeoutMs: 1_000, + }, + }); + + await expect(client.getRunnerScaleSet(4, 'linux')).resolves.toBeNull(); + + expect(registrationTokenRequests).toBe(1); + expect(adminConnectionRequests).toBe(4); + expect(actionsRequests).toBe(1); + }); + + it('does not replay JIT generation when its POST receives a transient failure', async () => { + let jitRequests = 0; + const fetchImplementation = vi.fn(async (input) => { + const url = requestUrl(input); + + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + if (url.pathname.endsWith('/generatejitconfig')) { + jitRequests += 1; + return jsonResponse({ message: 'temporarily unavailable' }, 503); + } + return new Response(null, { status: 500 }); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-access-token', + fetch: fetchImplementation, + retry: { + maxRetries: 4, + initialBackoffMs: 0, + maxBackoffMs: 0, + requestTimeoutMs: 1_000, + }, + }); + + await expect(client.generateJitRunnerConfig({ name: 'runner-71', workFolder: '_work' }, 42)).rejects.toMatchObject({ + status: 503, + }); + expect(jitRequests).toBe(1); + }); + + it('raises a typed HTTP error with Actions exception and request metadata', async () => { + const fixture = clientFixture( + () => + new Response( + JSON.stringify({ + typeName: 'Microsoft.TeamFoundation.DistributedTask.WebApi.AgentExistsException', + message: 'runner already exists', + }), + { + status: 409, + headers: { + ActivityId: 'activity-123', + 'Content-Type': 'application/json', + 'X-GitHub-Request-Id': 'github-456', + }, + }, + ), + ); + + const request = fixture.client.getRunner(71); + await expect(request).rejects.toBeInstanceOf(ScaleSetHttpError); + await expect(request).rejects.toMatchObject({ + code: SCALE_SET_ERROR_CODES.runnerExists, + status: 409, + activityId: 'activity-123', + githubRequestId: 'github-456', + exceptionName: 'Microsoft.TeamFoundation.DistributedTask.WebApi.AgentExistsException', + }); + }); + + it('uses the exact scale-set CRUD endpoints and legacy request body casing', async () => { + const fixture = clientFixture((url, init) => { + const method = init.method; + if (method === 'GET' && url.pathname.endsWith('/runnerscalesets')) { + return jsonResponse({ + count: url.searchParams.has('name') ? 1 : 2, + value: [ + { id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }, + { id: 12, name: 'windows', RunnerSetting: {} }, + ], + }); + } + if (method === 'GET' && url.pathname.endsWith('/runnerscalesets/11')) { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }); + } + if (method === 'POST') { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: true } }); + } + if (method === 'PATCH') { + return jsonResponse({ id: 11, name: 'linux', RunnerSetting: { disableUpdate: false } }); + } + if (method === 'DELETE') { + return new Response(null, { status: 204 }); + } + return new Response(null, { status: 500 }); + }); + const createInput = { + name: 'linux', + runnerGroupId: 4, + runnerSetting: { disableUpdate: true }, + }; + const updateInput = { + labels: [{ name: 'arm64' }], + runnerSetting: { disableUpdate: false }, + }; + + await expect(fixture.client.getRunnerScaleSet(4, 'linux')).resolves.toMatchObject({ + id: 11, + runnerSetting: { disableUpdate: true }, + }); + await expect(fixture.client.listRunnerScaleSets(4)).resolves.toHaveLength(2); + await expect(fixture.client.getRunnerScaleSetById(11)).resolves.toMatchObject({ id: 11 }); + await expect(fixture.client.createRunnerScaleSet(createInput)).resolves.toMatchObject({ id: 11 }); + await expect(fixture.client.updateRunnerScaleSet(11, updateInput)).resolves.toMatchObject({ + id: 11, + }); + await expect(fixture.client.deleteRunnerScaleSet(11)).resolves.toBeUndefined(); + + expect(createInput).toMatchObject({ labels: [{ name: 'linux', type: 'System' }] }); + expect(updateInput).toMatchObject({ labels: [{ name: 'arm64', type: 'System' }] }); + + const createRequest = fixture.serviceRequests.find(({ init }) => init.method === 'POST'); + const updateRequest = fixture.serviceRequests.find(({ init }) => init.method === 'PATCH'); + expect(createRequest).toBeDefined(); + expect(updateRequest).toBeDefined(); + + const createBody = JSON.parse(createRequest?.init.body as string) as Record; + expect(createBody).toMatchObject({ + name: 'linux', + runnerGroupId: 4, + labels: [{ name: 'linux', type: 'System' }], + RunnerSetting: { disableUpdate: true }, + }); + expect(createBody).not.toHaveProperty('runnerSetting'); + + const updateBody = JSON.parse(updateRequest?.init.body as string) as Record; + expect(updateBody).toMatchObject({ + labels: [{ name: 'arm64', type: 'System' }], + RunnerSetting: { disableUpdate: false }, + }); + expect(updateBody).not.toHaveProperty('runnerSetting'); + + expect(fixture.serviceRequests.map(({ url, init }) => [init.method, url.pathname])).toEqual([ + ['GET', '/tenant/123/_apis/runtime/runnerscalesets'], + ['GET', '/tenant/123/_apis/runtime/runnerscalesets'], + ['GET', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ['POST', '/tenant/123/_apis/runtime/runnerscalesets'], + ['PATCH', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ['DELETE', '/tenant/123/_apis/runtime/runnerscalesets/11'], + ]); + for (const { url } of fixture.serviceRequests) { + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + } + }); + + it('uses the exact runner-group, JIT, and agent endpoints', async () => { + const fixture = clientFixture((url, init) => { + if (url.pathname.endsWith('/runnergroups/')) { + return jsonResponse({ + count: 1, + value: [{ id: 8, name: 'default', size: 0, isDefaultGroup: true }], + }); + } + if (url.pathname.endsWith('/generatejitconfig')) { + return jsonResponse({ + runner: { id: 71, name: 'runner-71', runnerScaleSetId: 42 }, + encodedJITConfig: 'encoded-jit', + }); + } + if (init.method === 'GET' && url.pathname.endsWith('/agents/71')) { + return jsonResponse({ id: 71, name: 'runner-71', runnerScaleSetId: 42 }); + } + if (init.method === 'GET' && url.pathname.endsWith('/agents')) { + return jsonResponse({ + count: 1, + value: [{ id: 71, name: 'runner-71', runnerScaleSetId: 42 }], + }); + } + if (init.method === 'DELETE' && url.pathname.endsWith('/agents/71')) { + return new Response(null, { status: 204 }); + } + return new Response(null, { status: 500 }); + }); + + await expect(fixture.client.getRunnerGroupByName('default')).resolves.toMatchObject({ id: 8 }); + await expect( + fixture.client.generateJitRunnerConfig({ name: 'runner-71', workFolder: '_work' }, 42), + ).resolves.toEqual({ + runner: { id: 71, name: 'runner-71', runnerScaleSetId: 42 }, + encodedJITConfig: 'encoded-jit', + }); + await expect(fixture.client.getRunner(71)).resolves.toMatchObject({ id: 71 }); + await expect(fixture.client.getRunnerByName('runner-71')).resolves.toMatchObject({ id: 71 }); + await expect(fixture.client.removeRunner(71)).resolves.toBeUndefined(); + + expect(fixture.serviceRequests.map(({ url, init }) => [init.method, url.pathname])).toEqual([ + ['GET', '/tenant/123/_apis/runtime/runnergroups/'], + ['POST', '/tenant/123/_apis/runtime/runnerscalesets/42/generatejitconfig'], + ['GET', '/tenant/123/_apis/distributedtask/pools/0/agents/71'], + ['GET', '/tenant/123/_apis/distributedtask/pools/0/agents'], + ['DELETE', '/tenant/123/_apis/distributedtask/pools/0/agents/71'], + ]); + expect(fixture.serviceRequests[0].url.searchParams.get('groupName')).toBe('default'); + expect(fixture.serviceRequests[3].url.searchParams.get('agentName')).toBe('runner-71'); + expect(JSON.parse(fixture.serviceRequests[1].init.body as string)).toEqual({ + name: 'runner-71', + workFolder: '_work', + }); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/client.ts b/lambdas/libs/github-actions-scale-set/src/client.ts new file mode 100644 index 0000000000..ec6b98c805 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/client.ts @@ -0,0 +1,666 @@ +import { githubApiUrl, ParsedGitHubConfig, parseGitHubConfigUrl, runnerRegistrationTokenPath } from './config'; +import { ACTIONS_API_VERSION, RUNNER_ENDPOINT, RUNNER_GROUP_ENDPOINT, SCALE_SET_ENDPOINT } from './endpoints'; +import { ScaleSetProtocolError } from './errors'; +import { createRetryingFetch, executeRequest, HttpResult, parseJsonResponse } from './http'; +import { MessageSessionClient } from './message-session-client'; +import { + AccessTokenProvider, + GitHubActionsScaleSetClientOptions, + RunnerGroup, + GitHubRunnerReference, + RunnerReference, + RunnerScaleSet, + RunnerScaleSetJitRunnerConfig, + RunnerScaleSetJitRunnerSetting, + ScaleSetFetch, + ScaleSetRunnerState, + ScaleSetRequestOptions, + SystemInfo, +} from './types'; + +const ADMIN_TOKEN_REFRESH_SKEW_MS = 60_000; +const SUCCESS_STATUSES = Array.from({ length: 100 }, (_, index) => index + 200); + +interface RegistrationTokenResponse { + token?: string; + expires_at?: string; +} + +interface ActionsServiceAdminConnectionResponse { + url?: string; + token?: string; +} + +interface ActionsServiceAdminToken { + token: string; + expiresAt: Date; + url: string; +} + +interface RunnerScaleSetListResponse { + count: number; + value: RunnerScaleSet[]; +} + +interface RunnerGroupListResponse { + count: number; + value: RunnerGroup[]; +} + +interface RunnerReferenceListResponse { + count: number; + value: RunnerReference[]; +} + +interface GitHubRunnerListResponse { + total_count: number; + runners: GitHubRunnerReference[]; +} + +const MAX_GITHUB_RUNNER_PAGES = 100; + +interface ActionsRequestOptions extends ScaleSetRequestOptions { + query?: Record; + body?: unknown; + expectedStatuses: readonly number[]; + authorization?: string; +} + +function joinUrlPath(base: string, path: string): string { + if (base === '') { + if (path === '') { + return ''; + } + return path.startsWith('/') ? path : `/${path}`; + } + if (path === '') { + return base.replace(/\/+$/, ''); + } + return `${base.replace(/\/+$/, '')}${path.startsWith('/') ? '' : '/'}${path}`; +} + +export function actionsServiceUrl( + base: string, + path: string, + query: Record = {}, +): URL { + const [pathOnly, pathQuery = ''] = path.split('?', 2); + const result = new URL(joinUrlPath(base, pathOnly)); + const mergedQuery = new URLSearchParams(pathQuery); + + for (const [name, value] of Object.entries(query)) { + if (value !== undefined) { + mergedQuery.set(name, String(value)); + } + } + if (!mergedQuery.get('api-version')) { + mergedQuery.set('api-version', ACTIONS_API_VERSION); + } + result.search = mergedQuery.toString(); + return result; +} + +function encodeSystemUserAgent(systemInfo: SystemInfo): string { + return JSON.stringify({ + system: systemInfo.system ?? '', + version: systemInfo.version ?? '', + commit_sha: systemInfo.commitSha ?? '', + scale_set_id: systemInfo.scaleSetId ?? 0, + subsystem: systemInfo.subsystem ?? '', + build_version: '1.0.0', + build_commit_sha: '', + kind: 'scaleset', + }); +} + +function parseJwtExpiration(token: string): Date { + const parts = token.split('.'); + if (parts.length < 2) { + throw new ScaleSetProtocolError('Actions service admin token is not a JWT'); + } + + let claims: unknown; + try { + claims = JSON.parse(Buffer.from(parts[1], 'base64url').toString('utf8')) as unknown; + } catch (error) { + throw new ScaleSetProtocolError('failed to decode Actions service admin token claims', { + cause: error, + }); + } + + if ( + typeof claims !== 'object' || + claims === null || + !('exp' in claims) || + typeof claims.exp !== 'number' || + !Number.isFinite(claims.exp) + ) { + throw new ScaleSetProtocolError('Actions service admin token is missing a numeric exp claim'); + } + + return new Date(claims.exp * 1000); +} + +function applyDefaultLabelTypes(scaleSet: RunnerScaleSet): void { + for (const label of scaleSet.labels ?? []) { + label.type ||= 'System'; + } +} + +function ensureLabels(scaleSet: RunnerScaleSet): void { + if ((scaleSet.labels?.length ?? 0) > 0) { + return; + } + if (!scaleSet.name) { + throw new ScaleSetProtocolError('runner scale set must have a name or at least one label'); + } + scaleSet.labels = [{ name: scaleSet.name, type: 'System' }]; +} + +function runnerScaleSetRequestBody(scaleSet: RunnerScaleSet): Record { + const wire = { ...scaleSet } as Record; + delete wire.runnerSetting; + // The capital R is intentional and matches the Actions scale set wire contract. + wire.RunnerSetting = scaleSet.runnerSetting ?? {}; + return wire; +} + +function normalizeRunnerScaleSet(scaleSet: RunnerScaleSet | null): RunnerScaleSet | null { + if (scaleSet === null) { + return null; + } + + const wire = scaleSet as RunnerScaleSet & { RunnerSetting?: RunnerScaleSet['runnerSetting'] }; + if (wire.runnerSetting === undefined && wire.RunnerSetting !== undefined) { + wire.runnerSetting = wire.RunnerSetting; + } + delete wire.RunnerSetting; + return wire; +} + +/** A native-fetch client for the GitHub Actions runner scale set APIs. */ +export class GitHubActionsScaleSetClient { + private readonly config: ParsedGitHubConfig; + private readonly fetchImplementation: ScaleSetFetch; + private readonly adminConnectionFetchImplementation: ScaleSetFetch; + private readonly accessTokenProvider: AccessTokenProvider; + private readonly now: () => Date; + private readonly customUserAgent?: string; + private currentSystemInfo: SystemInfo; + private currentUserAgent: string; + private adminToken?: ActionsServiceAdminToken; + private adminTokenRefresh?: Promise; + + constructor(options: GitHubActionsScaleSetClientOptions) { + this.config = parseGitHubConfigUrl(options.gitHubConfigUrl, options.forceGhes); + const fetchImplementation = options.fetch ?? globalThis.fetch; + if (typeof fetchImplementation !== 'function') { + throw new TypeError('a fetch implementation is required'); + } + this.fetchImplementation = createRetryingFetch(fetchImplementation, options.retry); + // Upstream retries transient RemoteAuth propagation failures only for this + // bootstrap POST. Queue 401s must remain owned by session-token refresh, + // and other non-idempotent SDK requests must never use this opt-in wrapper. + this.adminConnectionFetchImplementation = createRetryingFetch(fetchImplementation, options.retry, { + additionalRetryStatuses: [401, 403], + additionalRetryMethods: ['POST'], + }); + + const hasPersonalAccessToken = + typeof options.personalAccessToken === 'string' && options.personalAccessToken !== ''; + const hasAccessTokenProvider = typeof options.accessTokenProvider === 'function'; + if (hasPersonalAccessToken === hasAccessTokenProvider) { + throw new TypeError('provide exactly one of personalAccessToken or accessTokenProvider'); + } + + this.accessTokenProvider = hasPersonalAccessToken + ? async () => options.personalAccessToken as string + : (options.accessTokenProvider as AccessTokenProvider); + this.now = options.now ?? (() => new Date()); + this.currentSystemInfo = { ...options.systemInfo }; + this.customUserAgent = options.userAgent; + this.currentUserAgent = options.userAgent ?? encodeSystemUserAgent(this.currentSystemInfo); + } + + get gitHubConfig(): ParsedGitHubConfig { + return { + ...this.config, + configUrl: new URL(this.config.configUrl), + }; + } + + get systemInfo(): SystemInfo { + return { ...this.currentSystemInfo }; + } + + setSystemInfo(systemInfo: SystemInfo): void { + this.currentSystemInfo = { ...systemInfo }; + if (this.customUserAgent === undefined) { + this.currentUserAgent = encodeSystemUserAgent(systemInfo); + } + } + + debugInfo(): string { + return JSON.stringify({ + system_info: this.currentUserAgent, + }); + } + + async getRunnerScaleSet( + runnerGroupId: number, + runnerScaleSetName: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + const { result, url } = await this.actionsRequest('GET', SCALE_SET_ENDPOINT, { + expectedStatuses: [200], + query: { runnerGroupId, name: runnerScaleSetName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + return null; + } + if (list.count !== 1) { + throw new ScaleSetProtocolError( + `multiple runner scale sets found with name ${JSON.stringify(runnerScaleSetName)}`, + ); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner scale set response count was 1 but value was empty'); + } + return normalizeRunnerScaleSet(list.value[0]); + } + + async listRunnerScaleSets(runnerGroupId: number, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', SCALE_SET_ENDPOINT, { + expectedStatuses: [200], + query: { runnerGroupId }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + return list.value.map((scaleSet) => normalizeRunnerScaleSet(scaleSet) as RunnerScaleSet); + } + + async getRunnerScaleSetById( + runnerScaleSetId: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`; + const { result, url } = await this.actionsRequest('GET', path, { + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'GET', url)); + } + + async getRunnerGroupByName(runnerGroupName: string, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', `/${RUNNER_GROUP_ENDPOINT}`, { + expectedStatuses: [200], + query: { groupName: runnerGroupName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + throw new ScaleSetProtocolError(`no runner group found with name ${JSON.stringify(runnerGroupName)}`); + } + if (list.count !== 1) { + throw new ScaleSetProtocolError(`multiple runner groups found with name ${JSON.stringify(runnerGroupName)}`); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner group response count was 1 but value was empty'); + } + return list.value[0]; + } + + async createRunnerScaleSet(scaleSet: RunnerScaleSet, options: ScaleSetRequestOptions = {}): Promise { + ensureLabels(scaleSet); + applyDefaultLabelTypes(scaleSet); + const { result, url } = await this.actionsRequest('POST', SCALE_SET_ENDPOINT, { + body: runnerScaleSetRequestBody(scaleSet), + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'POST', url)) as RunnerScaleSet; + } + + async updateRunnerScaleSet( + runnerScaleSetId: number, + scaleSet: RunnerScaleSet, + options: ScaleSetRequestOptions = {}, + ): Promise { + applyDefaultLabelTypes(scaleSet); + const path = `${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`; + const { result, url } = await this.actionsRequest('PATCH', path, { + body: runnerScaleSetRequestBody(scaleSet), + expectedStatuses: [200], + signal: options.signal, + }); + return normalizeRunnerScaleSet(parseJsonResponse(result, 'PATCH', url)) as RunnerScaleSet; + } + + async deleteRunnerScaleSet(runnerScaleSetId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.actionsRequest('DELETE', `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}`, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + async generateJitRunnerConfig( + setting: RunnerScaleSetJitRunnerSetting, + runnerScaleSetId: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${runnerScaleSetId}/generatejitconfig`; + const { result, url } = await this.actionsRequest('POST', path, { + body: setting, + expectedStatuses: [200], + signal: options.signal, + }); + return parseJsonResponse(result, 'POST', url); + } + + async getRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { + const path = `/${RUNNER_ENDPOINT}/${runnerId}`; + const { result, url } = await this.actionsRequest('GET', path, { + expectedStatuses: [200], + signal: options.signal, + }); + return parseJsonResponse(result, 'GET', url); + } + + async listRunners(options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', RUNNER_ENDPOINT, { + expectedStatuses: [200], + signal: options.signal, + }); + return parseJsonResponse(result, 'GET', url).value; + } + + async listGitHubRunners(options: ScaleSetRequestOptions = {}): Promise { + const runners: GitHubRunnerReference[] = []; + for (let page = 1; page <= MAX_GITHUB_RUNNER_PAGES; page += 1) { + const url = githubApiUrl(this.config, `${this.runnerListPath()}?per_page=100&page=${page}`); + const accessToken = await this.getAccessToken(); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'GET', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${accessToken}`, + 'User-Agent': this.currentUserAgent, + 'X-GitHub-Api-Version': '2022-11-28', + }, + signal: options.signal, + }, + [200], + ); + const response = parseJsonResponse(result, 'GET', url); + if (!Array.isArray(response.runners)) { + throw new ScaleSetProtocolError('GitHub runner list response is missing runners'); + } + runners.push(...response.runners); + if (response.runners.length < 100 || runners.length >= response.total_count) return runners; + } + throw new ScaleSetProtocolError(`GitHub runner inventory exceeded ${MAX_GITHUB_RUNNER_PAGES} pages`); + } + + /** Fetch one runner directly from GitHub immediately before a destructive action. */ + async getGitHubRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { + const url = githubApiUrl(this.config, `${this.runnerListPath()}/${runnerId}`); + const accessToken = await this.getAccessToken(); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'GET', + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${accessToken}`, + 'User-Agent': this.currentUserAgent, + 'X-GitHub-Api-Version': '2022-11-28', + }, + signal: options.signal, + }, + [200, 404], + ); + if (result.response.status === 404) return null; + return parseJsonResponse(result, 'GET', url); + } + + async listScaleSetRunnerStates( + runnerScaleSetId: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + const [actionsRunners, githubRunners] = await Promise.all([ + this.listRunners(options), + this.listGitHubRunners(options), + ]); + const githubById = new Map(); + const duplicateIds = new Set(); + for (const runner of githubRunners) { + if (githubById.has(runner.id)) duplicateIds.add(runner.id); + else githubById.set(runner.id, runner); + } + return actionsRunners + .filter((runner) => runner.runnerScaleSetId === runnerScaleSetId) + .map((runner) => { + const githubRunner = duplicateIds.has(runner.id) ? undefined : githubById.get(runner.id); + const exact = githubRunner?.name === runner.name; + const status = + exact && (githubRunner.status === 'online' || githubRunner.status === 'offline') + ? githubRunner.status + : 'unknown'; + return { + runnerId: runner.id, + runnerName: runner.name, + scaleSetId: runner.runnerScaleSetId, + status, + busy: exact && typeof githubRunner.busy === 'boolean' ? githubRunner.busy : undefined, + }; + }); + } + + async getRunnerByName(runnerName: string, options: ScaleSetRequestOptions = {}): Promise { + const { result, url } = await this.actionsRequest('GET', RUNNER_ENDPOINT, { + expectedStatuses: [200], + query: { agentName: runnerName }, + signal: options.signal, + }); + const list = parseJsonResponse(result, 'GET', url); + + if (list.count === 0) { + return null; + } + if (list.count !== 1) { + throw new ScaleSetProtocolError(`multiple runners found with name ${JSON.stringify(runnerName)}`); + } + if (list.value.length === 0) { + throw new ScaleSetProtocolError('runner response count was 1 but value was empty'); + } + return list.value[0]; + } + + async removeRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.actionsRequest('DELETE', `/${RUNNER_ENDPOINT}/${runnerId}`, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + async createMessageSessionClient( + runnerScaleSetId: number, + owner: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + return MessageSessionClient.create({ + runnerScaleSetId, + owner, + fetchImplementation: this.fetchImplementation, + userAgent: () => this.currentUserAgent, + actionsRequest: (method, path, requestOptions) => this.actionsRequest(method, path, requestOptions), + signal: options.signal, + }); + } + + /** Alias matching the upstream Go client's factory name. */ + async messageSessionClient( + runnerScaleSetId: number, + owner: string, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.createMessageSessionClient(runnerScaleSetId, owner, options); + } + + private async actionsRequest( + method: string, + path: string, + options: ActionsRequestOptions, + ): Promise<{ result: HttpResult; url: URL }> { + const adminToken = await this.getAdminToken(options.signal); + const url = actionsServiceUrl(adminToken.url, path, options.query); + const headers: Record = { + 'Content-Type': 'application/json', + Authorization: options.authorization ?? `Bearer ${adminToken.token}`, + 'User-Agent': this.currentUserAgent, + }; + const body = options.body === undefined ? undefined : JSON.stringify(options.body); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method, + headers, + body, + signal: options.signal, + }, + options.expectedStatuses, + ); + return { result, url }; + } + + private async getAdminToken(signal?: AbortSignal): Promise { + if (this.adminTokenIsUsable(this.adminToken)) { + return this.adminToken; + } + + if (this.adminTokenRefresh === undefined) { + this.adminTokenRefresh = this.refreshAdminToken(signal).finally(() => { + this.adminTokenRefresh = undefined; + }); + } + return this.adminTokenRefresh; + } + + private adminTokenIsUsable(token?: ActionsServiceAdminToken): token is ActionsServiceAdminToken { + return token !== undefined && this.now().getTime() + ADMIN_TOKEN_REFRESH_SKEW_MS < token.expiresAt.getTime(); + } + + private async refreshAdminToken(signal?: AbortSignal): Promise { + const registrationToken = await this.getRunnerRegistrationToken(signal); + const adminConnection = await this.getActionsServiceAdminConnection(registrationToken, signal); + const refreshedToken: ActionsServiceAdminToken = { + token: adminConnection.token, + expiresAt: parseJwtExpiration(adminConnection.token), + url: adminConnection.url, + }; + this.adminToken = refreshedToken; + return refreshedToken; + } + + private async getRunnerRegistrationToken(signal?: AbortSignal): Promise { + const accessToken = await this.getAccessToken(); + const url = githubApiUrl(this.config, runnerRegistrationTokenPath(this.config)); + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'POST', + headers: { + 'Content-Type': 'application/vnd.github.v3+json', + Authorization: `Bearer ${accessToken}`, + 'User-Agent': this.currentUserAgent, + }, + body: '', + signal, + }, + [201], + ); + const response = parseJsonResponse(result, 'POST', url); + if (!response.token) { + throw new ScaleSetProtocolError('runner registration token response is missing token'); + } + return response.token; + } + + private async getAccessToken(): Promise { + const providedToken = await this.accessTokenProvider(); + const accessToken = typeof providedToken === 'string' ? providedToken : providedToken.token; + if (accessToken === '') throw new ScaleSetProtocolError('access token provider returned an empty token'); + return accessToken; + } + + private runnerListPath(): string { + switch (this.config.scope) { + case 'organization': + return `/orgs/${this.config.organization}/actions/runners`; + case 'repository': + return `/repos/${this.config.organization}/${this.config.repository}/actions/runners`; + case 'enterprise': + return `/enterprises/${this.config.enterprise}/actions/runners`; + } + } + + private async getActionsServiceAdminConnection( + registrationToken: string, + signal?: AbortSignal, + ): Promise<{ url: string; token: string }> { + const url = githubApiUrl(this.config, '/actions/runner-registration'); + const result = await executeRequest( + this.adminConnectionFetchImplementation, + url, + { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `RemoteAuth ${registrationToken}`, + 'User-Agent': this.currentUserAgent, + }, + body: JSON.stringify({ + url: this.config.configUrl.toString(), + runner_event: 'register', + }), + signal, + }, + SUCCESS_STATUSES, + ); + const response = parseJsonResponse(result, 'POST', url); + if (!response.url) { + throw new ScaleSetProtocolError('Actions service admin connection is missing url'); + } + if (!response.token) { + throw new ScaleSetProtocolError('Actions service admin connection is missing token'); + } + let actionsServiceUrl: URL; + try { + actionsServiceUrl = new URL(response.url); + } catch (error) { + throw new ScaleSetProtocolError('Actions service admin connection contains an invalid url', { cause: error }); + } + if ( + actionsServiceUrl.protocol !== 'https:' || + actionsServiceUrl.username || + actionsServiceUrl.password || + actionsServiceUrl.search || + actionsServiceUrl.hash + ) { + throw new ScaleSetProtocolError( + 'Actions service admin connection url must be HTTPS and contain no credentials, query, or fragment', + ); + } + return { url: actionsServiceUrl.toString().replace(/\/$/, ''), token: response.token }; + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/config.test.ts b/lambdas/libs/github-actions-scale-set/src/config.test.ts new file mode 100644 index 0000000000..35fc866066 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/config.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, it } from 'vitest'; + +import { InvalidGitHubConfigUrlError, parseGitHubConfigUrl } from './config'; + +describe('GitHub configuration URL parsing', () => { + it('requires HTTPS for GitHub.com and GHES configuration URLs', () => { + expect(() => parseGitHubConfigUrl('http://github.com/example')).toThrow(InvalidGitHubConfigUrlError); + expect(() => parseGitHubConfigUrl('http://github.example.com/example', true)).toThrow(/should be HTTPS/); + }); + + it('continues to accept an HTTPS GitHub configuration URL', () => { + expect(parseGitHubConfigUrl('https://github.com/example')).toMatchObject({ + scope: 'organization', + organization: 'example', + isHosted: true, + }); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/config.ts b/lambdas/libs/github-actions-scale-set/src/config.ts new file mode 100644 index 0000000000..9007167dec --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/config.ts @@ -0,0 +1,117 @@ +export const GITHUB_SCOPES = { + enterprise: 'enterprise', + organization: 'organization', + repository: 'repository', +} as const; + +export type GitHubScope = (typeof GITHUB_SCOPES)[keyof typeof GITHUB_SCOPES]; + +export interface ParsedGitHubConfig { + configUrl: URL; + scope: GitHubScope; + enterprise?: string; + organization?: string; + repository?: string; + isHosted: boolean; +} + +export class InvalidGitHubConfigUrlError extends Error { + constructor(configUrl: string, options?: ErrorOptions) { + super( + `${JSON.stringify(configUrl)}: invalid config URL, should be HTTPS and point to an enterprise, org, or repository`, + options, + ); + this.name = 'InvalidGitHubConfigUrlError'; + } +} + +function environmentForcesGhes(): boolean { + return ( + typeof process !== 'undefined' && Object.prototype.hasOwnProperty.call(process.env, 'GITHUB_ACTIONS_FORCE_GHES') + ); +} + +function isHostedGitHubUrl(configUrl: URL, forceGhes?: boolean): boolean { + if (forceGhes ?? environmentForcesGhes()) { + return false; + } + + const host = configUrl.host.toLowerCase(); + return host === 'github.com' || host === 'www.github.com' || host === 'github.localhost' || host.endsWith('.ghe.com'); +} + +/** Parse a repository, organization, or enterprise registration URL. */ +export function parseGitHubConfigUrl(configUrl: string, forceGhes?: boolean): ParsedGitHubConfig { + let parsedUrl: URL; + try { + parsedUrl = new URL(configUrl.trim().replace(/\/+$/, '')); + } catch (error) { + throw new InvalidGitHubConfigUrlError(configUrl, { cause: error }); + } + + if (parsedUrl.protocol !== 'https:') { + throw new InvalidGitHubConfigUrlError(configUrl); + } + + const pathParts = parsedUrl.pathname.replace(/^\/+|\/+$/g, '').split('/'); + const isHosted = isHostedGitHubUrl(parsedUrl, forceGhes); + + if (pathParts.length === 1 && pathParts[0] !== '') { + parsedUrl.pathname = `/${pathParts[0]}`; + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.organization, + organization: pathParts[0], + isHosted, + }; + } + + if (pathParts.length === 2 && pathParts.every((part) => part !== '')) { + parsedUrl.pathname = `/${pathParts.join('/')}`; + if (pathParts[0].toLowerCase() === 'enterprises') { + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.enterprise, + enterprise: pathParts[1], + isHosted, + }; + } + + return { + configUrl: parsedUrl, + scope: GITHUB_SCOPES.repository, + organization: pathParts[0], + repository: pathParts[1], + isHosted, + }; + } + + throw new InvalidGitHubConfigUrlError(configUrl); +} + +/** Build a GitHub REST API URL for GitHub.com, ghe.com, or GHES. */ +export function githubApiUrl(config: ParsedGitHubConfig, path: string): URL { + const result = new URL(config.configUrl.origin); + const normalizedPath = path.startsWith('/') ? path : `/${path}`; + + if (config.isHosted) { + result.host = + config.configUrl.host.toLowerCase() === 'www.github.com' ? 'api.github.com' : `api.${config.configUrl.host}`; + result.pathname = normalizedPath; + return result; + } + + result.pathname = `/api/v3${normalizedPath}`; + return result; +} + +export function runnerRegistrationTokenPath(config: ParsedGitHubConfig): string { + switch (config.scope) { + case GITHUB_SCOPES.organization: + return `/orgs/${config.organization}/actions/runners/registration-token`; + case GITHUB_SCOPES.enterprise: + return `/enterprises/${config.enterprise}/actions/runners/registration-token`; + case GITHUB_SCOPES.repository: + return `/repos/${config.organization}/${config.repository}/actions/runners/registration-token`; + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/endpoints.ts b/lambdas/libs/github-actions-scale-set/src/endpoints.ts new file mode 100644 index 0000000000..d409f73704 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/endpoints.ts @@ -0,0 +1,4 @@ +export const RUNNER_ENDPOINT = '_apis/distributedtask/pools/0/agents'; +export const SCALE_SET_ENDPOINT = '_apis/runtime/runnerscalesets'; +export const RUNNER_GROUP_ENDPOINT = '_apis/runtime/runnergroups/'; +export const ACTIONS_API_VERSION = '6.0-preview'; diff --git a/lambdas/libs/github-actions-scale-set/src/errors.ts b/lambdas/libs/github-actions-scale-set/src/errors.ts new file mode 100644 index 0000000000..860e4aaa7d --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/errors.ts @@ -0,0 +1,171 @@ +export const SCALE_SET_ERROR_CODES = { + badRequest: 'BAD_REQUEST', + conflict: 'CONFLICT', + jobStillRunning: 'JOB_STILL_RUNNING', + messageQueueTokenExpired: 'MESSAGE_QUEUE_TOKEN_EXPIRED', + notFound: 'NOT_FOUND', + runnerExists: 'RUNNER_EXISTS', + runnerNotFound: 'RUNNER_NOT_FOUND', + unauthorized: 'UNAUTHORIZED', + unexpectedStatus: 'UNEXPECTED_STATUS', +} as const; + +export type ScaleSetErrorCode = (typeof SCALE_SET_ERROR_CODES)[keyof typeof SCALE_SET_ERROR_CODES]; + +export interface ScaleSetHttpErrorDetails { + method: string; + url: string; + status: number; + statusText: string; + headers: Headers; + responseBody: string; + code?: ScaleSetErrorCode; + cause?: unknown; +} + +interface ActionsException { + typeName?: unknown; + message?: unknown; +} + +export function redactUrlForError(value: string): string { + try { + const url = new URL(value); + url.username = ''; + url.password = ''; + url.search = ''; + url.hash = ''; + return url.toString(); + } catch { + return ''; + } +} + +function statusErrorCode(status: number): ScaleSetErrorCode { + switch (status) { + case 400: + return SCALE_SET_ERROR_CODES.badRequest; + case 401: + return SCALE_SET_ERROR_CODES.unauthorized; + case 404: + return SCALE_SET_ERROR_CODES.notFound; + case 409: + return SCALE_SET_ERROR_CODES.conflict; + default: + return SCALE_SET_ERROR_CODES.unexpectedStatus; + } +} + +function exceptionErrorCode(typeName?: string): ScaleSetErrorCode | undefined { + if (typeName?.includes('AgentExistsException')) { + return SCALE_SET_ERROR_CODES.runnerExists; + } + if (typeName?.includes('AgentNotFoundException')) { + return SCALE_SET_ERROR_CODES.runnerNotFound; + } + if (typeName?.includes('JobStillRunningException')) { + return SCALE_SET_ERROR_CODES.jobStillRunning; + } + return undefined; +} + +function parseActionsException(responseBody: string): { typeName?: string; message?: string } { + if (responseBody === '') { + return {}; + } + + try { + const parsed = JSON.parse(responseBody) as ActionsException; + return { + typeName: typeof parsed.typeName === 'string' ? parsed.typeName : undefined, + message: typeof parsed.message === 'string' ? parsed.message : undefined, + }; + } catch { + return {}; + } +} + +/** An unsuccessful HTTP response from either GitHub or the Actions service. */ +export class ScaleSetHttpError extends Error { + readonly code: ScaleSetErrorCode; + readonly status: number; + readonly statusText: string; + readonly method: string; + readonly url: string; + readonly activityId?: string; + readonly githubRequestId?: string; + readonly exceptionName?: string; + readonly responseBody: string; + + constructor(details: ScaleSetHttpErrorDetails) { + const safeUrl = redactUrlForError(details.url); + const exception = parseActionsException(details.responseBody); + const activityId = details.headers.get('ActivityId') ?? undefined; + const githubRequestId = details.headers.get('X-GitHub-Request-Id') ?? undefined; + const responseDescription = [details.status, details.statusText].filter(Boolean).join(' '); + const metadata = [ + `status=${JSON.stringify(responseDescription)}`, + activityId ? `activity_id=${JSON.stringify(activityId)}` : undefined, + githubRequestId ? `github_request_id=${JSON.stringify(githubRequestId)}` : undefined, + ] + .filter((part): part is string => part !== undefined) + .join(', '); + const responseMessage = exception.message ?? (details.responseBody || 'unknown error'); + const exceptionPrefix = exception.typeName ? `${exception.typeName}: ` : ''; + + super(`request ${details.method} ${safeUrl} failed (${metadata}): ${exceptionPrefix}${responseMessage}`, { + cause: details.cause, + }); + this.name = 'ScaleSetHttpError'; + this.code = details.code ?? exceptionErrorCode(exception.typeName) ?? statusErrorCode(details.status); + this.status = details.status; + this.statusText = details.statusText; + this.method = details.method; + this.url = safeUrl; + this.activityId = activityId; + this.githubRequestId = githubRequestId; + this.exceptionName = exception.typeName; + this.responseBody = details.responseBody; + } +} + +export class ScaleSetRequestError extends Error { + readonly method: string; + readonly url: string; + readonly attempts: number; + + constructor(method: string, url: string, cause: unknown, attempts = 1) { + const safeUrl = redactUrlForError(url); + super(`request ${method} ${safeUrl} failed before receiving a response after ${attempts} attempt(s)`, { cause }); + this.name = 'ScaleSetRequestError'; + this.method = method; + this.url = safeUrl; + this.attempts = attempts; + } +} + +export class ScaleSetRequestTimeoutError extends ScaleSetRequestError { + readonly timeoutMs: number; + + constructor(method: string, url: string, timeoutMs: number, attempts: number) { + super(method, url, new Error(`request attempt exceeded ${timeoutMs}ms`), attempts); + this.name = 'ScaleSetRequestTimeoutError'; + this.timeoutMs = timeoutMs; + } +} + +export class ScaleSetProtocolError extends Error { + readonly method?: string; + readonly url?: string; + + constructor(message: string, options: { method?: string; url?: string; cause?: unknown } = {}) { + super(message, { cause: options.cause }); + this.name = 'ScaleSetProtocolError'; + this.method = options.method; + this.url = options.url; + } +} + +export function isScaleSetHttpError(error: unknown): error is ScaleSetHttpError { + return error instanceof ScaleSetHttpError; +} diff --git a/lambdas/libs/github-actions-scale-set/src/http.test.ts b/lambdas/libs/github-actions-scale-set/src/http.test.ts new file mode 100644 index 0000000000..76f506d13f --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/http.test.ts @@ -0,0 +1,238 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { ScaleSetRequestError, ScaleSetRequestTimeoutError } from './errors'; +import { + createRetryingFetch, + DEFAULT_SCALE_SET_RETRY_OPTIONS, + executeRequest, + resolveScaleSetRetryOptions, +} from './http'; +import { ScaleSetFetch } from './types'; + +function okResponse(): Response { + return new Response('{"ok":true}', { status: 200 }); +} + +describe('retrying fetch', () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it('uses bounded defaults close to the upstream client and validates overrides', () => { + expect(resolveScaleSetRetryOptions()).toEqual({ + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 300_000, + }); + expect(DEFAULT_SCALE_SET_RETRY_OPTIONS).toEqual(resolveScaleSetRetryOptions()); + expect(() => resolveScaleSetRetryOptions({ maxRetries: -1 })).toThrow(/retry\.maxRetries/); + expect(() => resolveScaleSetRetryOptions({ maxRetries: 1.5 })).toThrow(/integer/); + expect(() => resolveScaleSetRetryOptions({ requestTimeoutMs: 0 })).toThrow(/requestTimeoutMs/); + }); + + it('retries a network error after deterministic exponential backoff', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi + .fn() + .mockRejectedValueOnce(new TypeError('socket closed')) + .mockResolvedValueOnce(okResponse()); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 25, + maxBackoffMs: 100, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.advanceTimersByTimeAsync(24); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + + await expect(request).resolves.toMatchObject({ status: 200 }); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + }); + + it('honors Retry-After for 429 responses and caps the wait at maxBackoffMs', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi + .fn() + .mockResolvedValueOnce( + new Response('{"message":"slow down"}', { + status: 429, + headers: { 'Retry-After': '120' }, + }), + ) + .mockResolvedValueOnce(okResponse()); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 10, + maxBackoffMs: 30_000, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.advanceTimersByTimeAsync(29_999); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(1); + + await expect(request).resolves.toMatchObject({ status: 200 }); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + }); + + it('retries 5xx responses only up to maxRetries and returns the final response', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn( + async () => + new Response('{"message":"unavailable"}', { + status: 503, + headers: { 'Retry-After': 'invalid' }, + }), + ); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 2, + initialBackoffMs: 10, + maxBackoffMs: 15, + requestTimeoutMs: 1_000, + }); + + const request = fetchWithRetry('https://actions.example/test'); + await vi.runAllTimersAsync(); + + await expect(request).resolves.toMatchObject({ status: 503 }); + expect(underlyingFetch).toHaveBeenCalledTimes(3); + }); + + it('does not retry a queue 401 so message-session refresh remains the owner', async () => { + const underlyingFetch = vi.fn(async () => new Response(null, { status: 401 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect(fetchWithRetry('https://queue.example/messages')).resolves.toMatchObject({ status: 401 }); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('does not replay a POST after a retryable HTTP response', async () => { + const underlyingFetch = vi.fn(async () => new Response(null, { status: 503 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect( + fetchWithRetry('https://actions.example/generatejitconfig', { method: 'POST' }), + ).resolves.toMatchObject({ status: 503 }); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('does not replay a POST carried by a Request after a network failure', async () => { + const underlyingFetch = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + await expect( + fetchWithRetry(new Request('https://actions.example/sessions', { method: 'POST' })), + ).rejects.toMatchObject({ + name: 'ScaleSetRequestError', + method: 'POST', + attempts: 1, + } satisfies Partial); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('times out each attempt and stops after the configured retry bound', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(() => new Promise(() => undefined)); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 10, + maxBackoffMs: 10, + requestTimeoutMs: 50, + }); + + const request = fetchWithRetry('https://actions.example/hangs'); + const rejection = expect(request).rejects.toMatchObject({ + name: 'ScaleSetRequestTimeoutError', + attempts: 2, + timeoutMs: 50, + } satisfies Partial); + await vi.advanceTimersByTimeAsync(50); + expect(underlyingFetch).toHaveBeenCalledOnce(); + await vi.advanceTimersByTimeAsync(10); + expect(underlyingFetch).toHaveBeenCalledTimes(2); + await vi.advanceTimersByTimeAsync(50); + + await rejection; + }); + + it('interrupts retry backoff immediately when the caller aborts', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(async () => new Response(null, { status: 503 })); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 4, + initialBackoffMs: 30_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 60_000, + }); + const controller = new AbortController(); + const request = fetchWithRetry('https://actions.example/test', { signal: controller.signal }); + await vi.advanceTimersByTimeAsync(0); + + controller.abort(new Error('caller cancelled')); + + await expect(request).rejects.toThrow('caller cancelled'); + expect(underlyingFetch).toHaveBeenCalledOnce(); + }); + + it('wraps an exhausted idempotent network failure with the final attempt count', async () => { + vi.useFakeTimers(); + const underlyingFetch = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const fetchWithRetry = createRetryingFetch(underlyingFetch, { + maxRetries: 1, + initialBackoffMs: 1, + maxBackoffMs: 1, + requestTimeoutMs: 100, + }); + + const request = fetchWithRetry('https://actions.example/test', { method: 'GET' }); + const rejection = expect(request).rejects.toMatchObject({ + name: 'ScaleSetRequestError', + method: 'GET', + attempts: 2, + } satisfies Partial); + await vi.runAllTimersAsync(); + + await rejection; + }); + + it('redacts signed query strings from request errors', async () => { + const fetchImplementation = vi.fn(async () => { + throw new TypeError('network unavailable'); + }); + const request = executeRequest( + fetchImplementation, + 'https://queue.example/messages?signature=do-not-log&token=also-secret', + { method: 'GET' }, + [200], + ); + + await expect(request).rejects.toMatchObject({ + url: 'https://queue.example/messages', + message: expect.not.stringContaining('do-not-log'), + } satisfies Partial); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/http.ts b/lambdas/libs/github-actions-scale-set/src/http.ts new file mode 100644 index 0000000000..ef5f47bab9 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/http.ts @@ -0,0 +1,327 @@ +import { + ScaleSetErrorCode, + ScaleSetHttpError, + ScaleSetProtocolError, + ScaleSetRequestError, + ScaleSetRequestTimeoutError, + redactUrlForError, +} from './errors'; +import { ScaleSetFetch, ScaleSetRetryOptions } from './types'; + +export interface ResolvedScaleSetRetryOptions { + maxRetries: number; + initialBackoffMs: number; + maxBackoffMs: number; + requestTimeoutMs: number; +} + +export const DEFAULT_SCALE_SET_RETRY_OPTIONS: Readonly = Object.freeze({ + maxRetries: 4, + initialBackoffMs: 1_000, + maxBackoffMs: 30_000, + requestTimeoutMs: 5 * 60_000, +}); + +export interface HttpResult { + response: Response; + body: string; +} + +interface RetryingFetchPolicy { + additionalRetryStatuses?: readonly number[]; + /** + * Escape hatch for a known-safe, operation-scoped wrapper. Non-idempotent + * methods are never retried by the default transport policy. + */ + additionalRetryMethods?: readonly string[]; +} + +const MAX_RESPONSE_BODY_BYTES = 1024 * 1024; +const IDEMPOTENT_RETRY_METHODS = new Set(['GET', 'HEAD', 'OPTIONS', 'PUT', 'DELETE']); + +function trimByteOrderMark(body: string): string { + return body.startsWith('\uFEFF') ? body.slice(1) : body; +} + +function boundedNumber(name: string, value: number, minimum: number, integer: boolean): number { + if (!Number.isFinite(value) || value < minimum || (integer && !Number.isInteger(value))) { + throw new TypeError(`${name} must be ${integer ? 'an integer' : 'a number'} greater than or equal to ${minimum}`); + } + return value; +} + +export function resolveScaleSetRetryOptions(options: ScaleSetRetryOptions = {}): ResolvedScaleSetRetryOptions { + return { + maxRetries: boundedNumber( + 'retry.maxRetries', + options.maxRetries ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.maxRetries, + 0, + true, + ), + initialBackoffMs: boundedNumber( + 'retry.initialBackoffMs', + options.initialBackoffMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.initialBackoffMs, + 0, + false, + ), + maxBackoffMs: boundedNumber( + 'retry.maxBackoffMs', + options.maxBackoffMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.maxBackoffMs, + 0, + false, + ), + requestTimeoutMs: boundedNumber( + 'retry.requestTimeoutMs', + options.requestTimeoutMs ?? DEFAULT_SCALE_SET_RETRY_OPTIONS.requestTimeoutMs, + 1, + false, + ), + }; +} + +function abortReason(signal: AbortSignal): unknown { + return signal.reason ?? new DOMException('The operation was aborted', 'AbortError'); +} + +function waitForRetry(delayMs: number, signal?: AbortSignal): Promise { + if (signal?.aborted) { + return Promise.reject(abortReason(signal)); + } + if (delayMs === 0) { + return Promise.resolve(); + } + + return new Promise((resolve, reject) => { + const timeout = setTimeout(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timeout); + reject(abortReason(signal as AbortSignal)); + }; + signal?.addEventListener('abort', onAbort, { once: true }); + }); +} + +function retryableStatus(status: number, additionalRetryStatuses: ReadonlySet): boolean { + return status === 429 || (status >= 500 && status <= 599) || additionalRetryStatuses.has(status); +} + +function requestMethod(input: RequestInput, init: RequestInit): string { + return (init.method ?? (input instanceof Request ? input.method : 'GET')).toUpperCase(); +} + +function exponentialBackoffMs(retryIndex: number, options: ResolvedScaleSetRetryOptions): number { + return Math.min(options.initialBackoffMs * 2 ** retryIndex, options.maxBackoffMs); +} + +function retryAfterMs(response: Response, options: ResolvedScaleSetRetryOptions): number | undefined { + const value = response.headers.get('Retry-After')?.trim(); + if (!value) { + return undefined; + } + + let delayMs: number; + if (/^\d+$/.test(value)) { + delayMs = Number(value) * 1_000; + } else { + const retryAt = Date.parse(value); + if (!Number.isFinite(retryAt)) { + return undefined; + } + delayMs = Math.max(0, retryAt - Date.now()); + } + return Math.min(delayMs, options.maxBackoffMs); +} + +async function fetchAttempt( + fetchImplementation: ScaleSetFetch, + input: RequestInput, + init: RequestInit, + options: ResolvedScaleSetRetryOptions, + attempt: number, +): Promise { + const method = requestMethod(input, init); + const url = input instanceof Request ? input.url : input.toString(); + const callerSignal = init.signal ?? undefined; + if (callerSignal?.aborted) { + throw abortReason(callerSignal); + } + + const attemptController = new AbortController(); + const forwardAbort = () => attemptController.abort(abortReason(callerSignal as AbortSignal)); + callerSignal?.addEventListener('abort', forwardAbort, { once: true }); + const timeoutError = new ScaleSetRequestTimeoutError(method, url, options.requestTimeoutMs, attempt); + const timeout = setTimeout(() => attemptController.abort(timeoutError), options.requestTimeoutMs); + + let onAttemptAbort: (() => void) | undefined; + const aborted = new Promise((_, reject) => { + onAttemptAbort = () => reject(abortReason(attemptController.signal)); + attemptController.signal.addEventListener('abort', onAttemptAbort, { once: true }); + }); + + try { + return await Promise.race([fetchImplementation(input, { ...init, signal: attemptController.signal }), aborted]); + } finally { + clearTimeout(timeout); + callerSignal?.removeEventListener('abort', forwardAbort); + if (onAttemptAbort !== undefined) { + attemptController.signal.removeEventListener('abort', onAttemptAbort); + } + } +} + +type RequestInput = Parameters[0]; + +/** Wrap native fetch with the bounded retry and timeout policy used by every SDK request. */ +export function createRetryingFetch( + fetchImplementation: ScaleSetFetch, + retryOptions: ScaleSetRetryOptions = {}, + policy: RetryingFetchPolicy = {}, +): ScaleSetFetch { + const options = resolveScaleSetRetryOptions(retryOptions); + const additionalRetryStatusSet = new Set(policy.additionalRetryStatuses ?? []); + const additionalRetryMethodSet = new Set((policy.additionalRetryMethods ?? []).map((method) => method.toUpperCase())); + + return async (input, init = {}) => { + const method = requestMethod(input, init); + const url = input instanceof Request ? input.url : input.toString(); + const maxRetries = + IDEMPOTENT_RETRY_METHODS.has(method) || additionalRetryMethodSet.has(method) ? options.maxRetries : 0; + + for (let retryIndex = 0; retryIndex <= maxRetries; retryIndex += 1) { + const attempt = retryIndex + 1; + try { + const response = await fetchAttempt(fetchImplementation, input, init, options, attempt); + if (!retryableStatus(response.status, additionalRetryStatusSet) || retryIndex === maxRetries) { + return response; + } + + const delayMs = retryAfterMs(response, options) ?? exponentialBackoffMs(retryIndex, options); + await response.body?.cancel().catch(() => undefined); + await waitForRetry(delayMs, init.signal ?? undefined); + } catch (error) { + if (init.signal?.aborted) { + throw abortReason(init.signal); + } + if (retryIndex === maxRetries) { + if (error instanceof ScaleSetRequestTimeoutError) { + throw error; + } + throw new ScaleSetRequestError(method, url, error, attempt); + } + await waitForRetry(exponentialBackoffMs(retryIndex, options), init.signal ?? undefined); + } + } + + throw new ScaleSetRequestError(method, url, new Error('retry loop exhausted'), maxRetries + 1); + }; +} + +export async function executeRequest( + fetchImplementation: ScaleSetFetch, + url: string | URL, + init: RequestInit, + expectedStatuses: readonly number[], + errorCode?: ScaleSetErrorCode | ((response: Response) => ScaleSetErrorCode | undefined), +): Promise { + const method = init.method ?? 'GET'; + const urlString = url.toString(); + const displayUrl = redactUrlForError(urlString); + let response: Response; + + try { + response = await fetchImplementation(url, { ...init, redirect: 'error' }); + } catch (error) { + if (init.signal?.aborted || (error instanceof Error && error.name === 'AbortError')) { + throw error; + } + if (error instanceof ScaleSetRequestError) { + throw error; + } + throw new ScaleSetRequestError(method, urlString, error); + } + + let body: string; + try { + const contentLength = response.headers.get('content-length'); + if (contentLength !== null && /^\d+$/.test(contentLength) && Number(contentLength) > MAX_RESPONSE_BODY_BYTES) { + throw new ScaleSetProtocolError( + `response body from ${method} ${displayUrl} exceeds ${MAX_RESPONSE_BODY_BYTES} bytes`, + { + method, + url: displayUrl, + }, + ); + } + if (response.body === null) { + body = ''; + } else { + const reader = response.body.getReader(); + const chunks: Uint8Array[] = []; + let total = 0; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + total += value.byteLength; + if (total > MAX_RESPONSE_BODY_BYTES) { + await reader.cancel().catch(() => undefined); + throw new ScaleSetProtocolError( + `response body from ${method} ${displayUrl} exceeds ${MAX_RESPONSE_BODY_BYTES} bytes`, + { method, url: displayUrl }, + ); + } + chunks.push(value); + } + const bytes = new Uint8Array(total); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + body = trimByteOrderMark(new TextDecoder('utf-8', { fatal: true }).decode(bytes)); + } + } catch (error) { + if (error instanceof ScaleSetProtocolError) throw error; + throw new ScaleSetProtocolError(`failed to read the response body from ${method} ${displayUrl}`, { + method, + url: displayUrl, + cause: error, + }); + } + + if (!expectedStatuses.includes(response.status)) { + throw new ScaleSetHttpError({ + method, + url: displayUrl, + status: response.status, + statusText: response.statusText, + headers: response.headers, + responseBody: body, + code: typeof errorCode === 'function' ? errorCode(response) : errorCode, + }); + } + + return { response, body }; +} + +export function parseJsonResponse(result: HttpResult, method: string, url: string | URL): T { + const urlString = redactUrlForError(url.toString()); + if (result.body === '') { + throw new ScaleSetProtocolError(`empty JSON response from ${method} ${urlString}`, { + method, + url: urlString, + }); + } + + try { + return JSON.parse(result.body) as T; + } catch (error) { + throw new ScaleSetProtocolError(`invalid JSON response from ${method} ${urlString}`, { + method, + url: urlString, + cause: error, + }); + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/index.ts b/lambdas/libs/github-actions-scale-set/src/index.ts new file mode 100644 index 0000000000..ab0aad76db --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/index.ts @@ -0,0 +1,29 @@ +export { + actionsServiceUrl, + GitHubActionsScaleSetClient as Client, + GitHubActionsScaleSetClient, + GitHubActionsScaleSetClient as ScaleSetClient, +} from './client'; +export { ACTIONS_API_VERSION, RUNNER_ENDPOINT, RUNNER_GROUP_ENDPOINT, SCALE_SET_ENDPOINT } from './endpoints'; +export { + GITHUB_SCOPES, + githubApiUrl, + InvalidGitHubConfigUrlError, + parseGitHubConfigUrl, + runnerRegistrationTokenPath, +} from './config'; +export type { GitHubScope, ParsedGitHubConfig } from './config'; +export { + isScaleSetHttpError, + redactUrlForError, + SCALE_SET_ERROR_CODES, + ScaleSetHttpError, + ScaleSetProtocolError, + ScaleSetRequestError, + ScaleSetRequestTimeoutError, +} from './errors'; +export type { ScaleSetErrorCode, ScaleSetHttpErrorDetails } from './errors'; +export { DEFAULT_SCALE_SET_RETRY_OPTIONS } from './http'; +export type { ResolvedScaleSetRetryOptions } from './http'; +export { HEADER_SCALE_SET_MAX_CAPACITY, MessageSessionClient } from './message-session-client'; +export * from './types'; diff --git a/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts b/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts new file mode 100644 index 0000000000..ed4e7d41ae --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/message-session-client.test.ts @@ -0,0 +1,281 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { GitHubActionsScaleSetClient } from './client'; +import { ScaleSetProtocolError } from './errors'; +import { HEADER_SCALE_SET_MAX_CAPACITY } from './message-session-client'; +import { RunnerScaleSetStatistic, ScaleSetFetch } from './types'; + +type RequestInput = Parameters[0]; +type RequestHandler = (url: URL, init: RequestInit) => Response | Promise; + +const statistics: RunnerScaleSetStatistic = { + totalAvailableJobs: 2, + totalAcquiredJobs: 1, + totalAssignedJobs: 1, + totalRunningJobs: 1, + totalRegisteredRunners: 2, + totalBusyRunners: 1, + totalIdleRunners: 1, +}; + +function requestUrl(input: RequestInput): URL { + if (input instanceof Request) { + return new URL(input.url); + } + return new URL(input.toString()); +} + +function jsonResponse(value: unknown, status = 200): Response { + return new Response(JSON.stringify(value), { + status, + headers: { 'Content-Type': 'application/json' }, + }); +} + +function actionsAdminToken(): string { + const payload = Buffer.from(JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 60 * 60 })).toString('base64url'); + return `header.${payload}.signature`; +} + +function sessionFixture(handler: RequestHandler) { + const requests: Array<{ url: URL; init: RequestInit }> = []; + const fetchImplementation = vi.fn(async (input, init = {}) => { + const url = requestUrl(input); + if (url.pathname === '/orgs/example/actions/runners/registration-token') { + return jsonResponse({ token: 'runner-registration-token' }, 201); + } + if (url.pathname === '/actions/runner-registration') { + return jsonResponse({ + url: 'https://actions.example/tenant/123/', + token: actionsAdminToken(), + }); + } + + requests.push({ url, init }); + return handler(url, init); + }); + const client = new GitHubActionsScaleSetClient({ + gitHubConfigUrl: 'https://github.com/example', + personalAccessToken: 'github-token', + fetch: fetchImplementation, + systemInfo: { system: 'unit-test', subsystem: 'listener' }, + }); + + return { client, fetchImplementation, requests }; +} + +function sessionResponse(token = 'queue-token') { + return { + sessionId: '11111111-1111-1111-1111-111111111111', + ownerName: 'listener-1', + runnerScaleSet: { id: 42, name: 'linux' }, + messageQueueUrl: 'https://queue.example/messages?existing=1', + messageQueueAccessToken: token, + statistics, + }; +} + +describe('MessageSessionClient', () => { + beforeEach(() => { + vi.restoreAllMocks(); + }); + + it('normalizes the capitalized RunnerSetting returned when creating a session', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse({ + ...sessionResponse(), + runnerScaleSet: { id: 42, name: 'linux', RunnerSetting: { disableUpdate: true } }, + }); + } + return new Response(null, { status: 500 }); + }); + + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + expect(session.session.runnerScaleSet?.runnerSetting).toEqual({ disableUpdate: true }); + expect(session.session.runnerScaleSet).not.toHaveProperty('RunnerSetting'); + }); + + it('maps a 202 poll to null and sends the queue capacity contract', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + return new Response(null, { status: 202 }); + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 17)).resolves.toBeNull(); + + expect(session.session.statistics).toEqual(statistics); + const queueRequest = fixture.requests.find(({ url }) => url.hostname === 'queue.example'); + expect(queueRequest).toBeDefined(); + expect(queueRequest?.url.toString()).toBe('https://queue.example/messages?existing=1'); + const headers = new Headers(queueRequest?.init.headers); + expect(headers.get('Accept')).toBe('application/json; api-version=6.0-preview'); + expect(headers.get('Authorization')).toBe('Bearer queue-token'); + expect(headers.get(HEADER_SCALE_SET_MAX_CAPACITY)).toBe('17'); + expect(headers.get('User-Agent')).toContain('"kind":"scaleset"'); + }); + + it('decodes known batched messages, ignores unknown types, acknowledges, and acquires jobs', async () => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + return jsonResponse({ + messageId: 19, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([ + { + messageType: 'JobAvailable', + runnerRequestId: 501, + acquireJobUrl: 'https://actions.example/acquire/501', + }, + { + messageType: 'FutureMessageType', + runnerRequestId: 999, + }, + { + messageType: 'JobCompleted', + runnerRequestId: 500, + runnerId: 71, + runnerName: 'runner-71', + result: 'Succeeded', + }, + ]), + }); + } + if (url.hostname === 'queue.example' && init.method === 'DELETE') { + return new Response(null, { status: 204 }); + } + if (url.pathname.endsWith('/runnerscalesets/42/acquirejobs') && init.method === 'POST') { + return jsonResponse({ count: 1, value: [501] }); + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + const message = await session.getMessage(18, 4); + expect(message).toMatchObject({ messageId: 19, statistics }); + expect(message?.jobAvailableMessages).toEqual([ + expect.objectContaining({ messageType: 'JobAvailable', runnerRequestId: 501 }), + ]); + expect(message?.jobCompletedMessages).toEqual([ + expect.objectContaining({ messageType: 'JobCompleted', runnerName: 'runner-71' }), + ]); + expect(message?.jobAssignedMessages).toEqual([]); + expect(message?.jobStartedMessages).toEqual([]); + + await expect(session.deleteMessage(19)).resolves.toBeUndefined(); + await expect(session.acquireJobs([501, 999])).resolves.toEqual([501]); + + const pollRequest = fixture.requests.find( + ({ url, init }) => url.hostname === 'queue.example' && init.method === 'GET', + ); + expect(pollRequest?.url.searchParams.get('lastMessageId')).toBe('18'); + + const ackRequest = fixture.requests.find( + ({ url, init }) => url.hostname === 'queue.example' && init.method === 'DELETE', + ); + expect(ackRequest?.url.pathname).toBe('/messages/19'); + expect(ackRequest?.url.searchParams.get('existing')).toBe('1'); + expect(new Headers(ackRequest?.init.headers).get('Authorization')).toBe('Bearer queue-token'); + + const acquireRequest = fixture.requests.find(({ url }) => url.pathname.endsWith('/acquirejobs')); + expect(acquireRequest?.url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets/42/acquirejobs'); + expect(acquireRequest?.url.searchParams.get('api-version')).toBe('6.0-preview'); + expect(new Headers(acquireRequest?.init.headers).get('Authorization')).toBe('Bearer queue-token'); + expect(JSON.parse(acquireRequest?.init.body as string)).toEqual([501, 999]); + }); + + it.each([ + ['message id', { messageId: 0, messageType: 'RunnerScaleSetJobMessages', statistics, body: '[]' }], + [ + 'statistics', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics: { ...statistics, totalAssignedJobs: -1 }, + body: '[]', + }, + ], + [ + 'runner request id', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([{ messageType: 'JobAvailable', runnerRequestId: 0 }]), + }, + ], + [ + 'runner identity', + { + messageId: 1, + messageType: 'RunnerScaleSetJobMessages', + statistics, + body: JSON.stringify([ + { messageType: 'JobCompleted', runnerRequestId: 1, runnerId: 2, runnerName: 'bad\nname' }, + ]), + }, + ], + ])('rejects malformed known message %s fields', async (_name, payload) => { + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse()); + } + if (url.hostname === 'queue.example' && init.method === 'GET') return jsonResponse(payload); + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 10)).rejects.toBeInstanceOf(ScaleSetProtocolError); + }); + + it('refreshes the message session once on a queue 401 and retries with the new token', async () => { + let refreshCount = 0; + let oldTokenPolls = 0; + let newTokenPolls = 0; + const fixture = sessionFixture((url, init) => { + if (url.pathname.endsWith('/runnerscalesets/42/sessions') && init.method === 'POST') { + return jsonResponse(sessionResponse('old-queue-token')); + } + if (url.pathname.includes('/runnerscalesets/42/sessions/') && init.method === 'PATCH') { + refreshCount += 1; + return jsonResponse({ + ...sessionResponse('new-queue-token'), + runnerScaleSet: { id: 42, name: 'linux', RunnerSetting: { disableUpdate: false } }, + }); + } + if (url.hostname === 'queue.example' && init.method === 'GET') { + const authorization = new Headers(init.headers).get('Authorization'); + if (authorization === 'Bearer old-queue-token') { + oldTokenPolls += 1; + return jsonResponse({ message: 'expired' }, 401); + } + if (authorization === 'Bearer new-queue-token') { + newTokenPolls += 1; + return new Response(null, { status: 202 }); + } + } + return new Response(null, { status: 500 }); + }); + const session = await fixture.client.createMessageSessionClient(42, 'listener-1'); + + await expect(session.getMessage(0, 10)).resolves.toBeNull(); + + expect(oldTokenPolls).toBe(1); + expect(newTokenPolls).toBe(1); + expect(refreshCount).toBe(1); + expect(session.session.messageQueueAccessToken).toBe('new-queue-token'); + expect(session.session.runnerScaleSet?.runnerSetting).toEqual({ disableUpdate: false }); + expect(session.session.runnerScaleSet).not.toHaveProperty('RunnerSetting'); + }); +}); diff --git a/lambdas/libs/github-actions-scale-set/src/message-session-client.ts b/lambdas/libs/github-actions-scale-set/src/message-session-client.ts new file mode 100644 index 0000000000..130e68482e --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/message-session-client.ts @@ -0,0 +1,467 @@ +import { SCALE_SET_ERROR_CODES, ScaleSetHttpError, ScaleSetProtocolError } from './errors'; +import { executeRequest, HttpResult, parseJsonResponse } from './http'; +import { SCALE_SET_ENDPOINT } from './endpoints'; +import { + JobAssigned, + JobAvailable, + JobCompleted, + JobStarted, + MESSAGE_TYPES, + RunnerScaleSetMessage, + RunnerScaleSet, + RunnerScaleSetSession, + RunnerScaleSetStatistic, + ScaleSetFetch, + ScaleSetRequestOptions, +} from './types'; + +export const HEADER_SCALE_SET_MAX_CAPACITY = 'X-ScaleSetMaxCapacity'; + +interface ActionsRequestOptions extends ScaleSetRequestOptions { + body?: unknown; + expectedStatuses: readonly number[]; + authorization?: string; +} + +interface MessageSessionClientCreateOptions extends ScaleSetRequestOptions { + runnerScaleSetId: number; + owner: string; + fetchImplementation: ScaleSetFetch; + userAgent: () => string; + actionsRequest: ( + method: string, + path: string, + options: ActionsRequestOptions, + ) => Promise<{ result: HttpResult; url: URL }>; +} + +interface RunnerScaleSetMessageResponse { + messageId: number; + messageType: string; + body?: string; + statistics?: RunnerScaleSetStatistic | null; +} + +interface AcquireJobsResponse { + count: number; + value: number[]; +} + +interface JobMessageType { + messageType?: unknown; +} + +const STATISTIC_FIELDS = [ + 'totalAvailableJobs', + 'totalAcquiredJobs', + 'totalAssignedJobs', + 'totalRunningJobs', + 'totalRegisteredRunners', + 'totalBusyRunners', + 'totalIdleRunners', +] as const satisfies readonly (keyof RunnerScaleSetStatistic)[]; + +function positiveInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || (value as number) <= 0) { + throw new ScaleSetProtocolError(`${field} must be a positive integer`); + } + return value as number; +} + +function validateStatistics(value: unknown): RunnerScaleSetStatistic | null { + if (value === null || value === undefined) return null; + if (typeof value !== 'object' || Array.isArray(value)) { + throw new ScaleSetProtocolError('runner scale set statistics must be an object'); + } + for (const field of STATISTIC_FIELDS) { + const statistic = (value as Record)[field]; + if (!Number.isSafeInteger(statistic) || (statistic as number) < 0) { + throw new ScaleSetProtocolError(`statistics.${field} must be a non-negative integer`); + } + } + return value as RunnerScaleSetStatistic; +} + +function validateKnownJobMessage(rawMessage: Record, messageType: string): void { + positiveInteger(rawMessage.runnerRequestId, `${messageType}.runnerRequestId`); + if (messageType === MESSAGE_TYPES.jobStarted || messageType === MESSAGE_TYPES.jobCompleted) { + positiveInteger(rawMessage.runnerId, `${messageType}.runnerId`); + if ( + typeof rawMessage.runnerName !== 'string' || + rawMessage.runnerName.length === 0 || + rawMessage.runnerName.length > 256 || + hasAsciiControlCharacter(rawMessage.runnerName) + ) { + throw new ScaleSetProtocolError(`${messageType}.runnerName is invalid`); + } + } +} + +function hasAsciiControlCharacter(value: string): boolean { + return [...value].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }); +} + +function normalizeRunnerScaleSet(scaleSet: RunnerScaleSet | null | undefined): RunnerScaleSet | null | undefined { + if (scaleSet === null || scaleSet === undefined) { + return scaleSet; + } + + const wire = scaleSet as RunnerScaleSet & { RunnerSetting?: RunnerScaleSet['runnerSetting'] }; + if (wire.runnerSetting === undefined && wire.RunnerSetting !== undefined) { + wire.runnerSetting = wire.RunnerSetting; + } + delete wire.RunnerSetting; + return wire; +} + +function normalizeSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + session.runnerScaleSet = normalizeRunnerScaleSet(session.runnerScaleSet); + return session; +} + +function cloneSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + return { + ...session, + runnerScaleSet: + session.runnerScaleSet === undefined || session.runnerScaleSet === null + ? session.runnerScaleSet + : { + ...session.runnerScaleSet, + labels: session.runnerScaleSet.labels?.map((label) => ({ ...label })), + runnerSetting: + session.runnerScaleSet.runnerSetting === undefined + ? undefined + : { ...session.runnerScaleSet.runnerSetting }, + statistics: + session.runnerScaleSet.statistics === undefined || session.runnerScaleSet.statistics === null + ? session.runnerScaleSet.statistics + : { ...session.runnerScaleSet.statistics }, + }, + statistics: + session.statistics === undefined || session.statistics === null ? session.statistics : { ...session.statistics }, + }; +} + +function validateSession(session: RunnerScaleSetSession): RunnerScaleSetSession { + if (!session.sessionId) { + throw new ScaleSetProtocolError('message session response is missing sessionId'); + } + if (!session.messageQueueUrl) { + throw new ScaleSetProtocolError('message session response is missing messageQueueUrl'); + } + if (!session.messageQueueAccessToken) { + throw new ScaleSetProtocolError('message session response is missing messageQueueAccessToken'); + } + let queueUrl: URL; + try { + queueUrl = new URL(session.messageQueueUrl); + } catch (error) { + throw new ScaleSetProtocolError('message session response contains an invalid messageQueueUrl', { cause: error }); + } + if (queueUrl.protocol !== 'https:' || queueUrl.username || queueUrl.password || queueUrl.hash) { + throw new ScaleSetProtocolError( + 'message session messageQueueUrl must be HTTPS and contain no credentials or fragment', + ); + } + return session; +} + +function parseRunnerScaleSetMessage(result: HttpResult, url: URL): RunnerScaleSetMessage { + const response = parseJsonResponse(result, 'GET', url); + positiveInteger(response.messageId, 'messageId'); + if (response.messageType !== 'RunnerScaleSetJobMessages') { + throw new ScaleSetProtocolError(`unsupported message type: ${response.messageType}`); + } + + let batchedMessages: unknown[] = []; + if (response.body) { + try { + const parsed = JSON.parse(response.body) as unknown; + if (!Array.isArray(parsed)) { + throw new TypeError('message body is not an array'); + } + batchedMessages = parsed; + if (batchedMessages.length > 50) { + throw new TypeError('message body contains more than 50 entries'); + } + } catch (error) { + throw new ScaleSetProtocolError('failed to unmarshal batched runner scale set messages', { + cause: error, + }); + } + } + + const message: RunnerScaleSetMessage = { + messageId: response.messageId, + statistics: validateStatistics(response.statistics), + jobAvailableMessages: [], + jobAssignedMessages: [], + jobStartedMessages: [], + jobCompletedMessages: [], + }; + + for (const rawMessage of batchedMessages) { + if (typeof rawMessage !== 'object' || rawMessage === null) { + throw new ScaleSetProtocolError('runner scale set job message is not an object'); + } + const messageType = (rawMessage as JobMessageType).messageType; + if (typeof messageType !== 'string') { + throw new ScaleSetProtocolError('runner scale set job message is missing messageType'); + } + switch (messageType) { + case MESSAGE_TYPES.jobAvailable: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobAvailableMessages.push(rawMessage as JobAvailable); + break; + case MESSAGE_TYPES.jobAssigned: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobAssignedMessages.push(rawMessage as JobAssigned); + break; + case MESSAGE_TYPES.jobStarted: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobStartedMessages.push(rawMessage as JobStarted); + break; + case MESSAGE_TYPES.jobCompleted: + validateKnownJobMessage(rawMessage as Record, messageType); + message.jobCompletedMessages.push(rawMessage as JobCompleted); + break; + default: + // The upstream client ignores unknown job message types for forward compatibility. + break; + } + } + + return message; +} + +/** A message queue session scoped to one runner scale set. */ +export class MessageSessionClient { + private readonly runnerScaleSetId: number; + private readonly fetchImplementation: ScaleSetFetch; + private readonly userAgent: () => string; + private readonly actionsRequest: MessageSessionClientCreateOptions['actionsRequest']; + private currentSession: RunnerScaleSetSession; + private sessionRefresh?: Promise; + + private constructor(options: MessageSessionClientCreateOptions, session: RunnerScaleSetSession) { + this.runnerScaleSetId = options.runnerScaleSetId; + this.fetchImplementation = options.fetchImplementation; + this.userAgent = options.userAgent; + this.actionsRequest = options.actionsRequest; + this.currentSession = session; + } + + static async create(options: MessageSessionClientCreateOptions): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${options.runnerScaleSetId}/sessions`; + const { result, url } = await options.actionsRequest('POST', path, { + body: { ownerName: options.owner }, + expectedStatuses: [200], + signal: options.signal, + }); + const session = validateSession(normalizeSession(parseJsonResponse(result, 'POST', url))); + return new MessageSessionClient(options, session); + } + + /** A defensive snapshot of the current session and its latest statistics. */ + get session(): RunnerScaleSetSession { + return cloneSession(this.currentSession); + } + + getSession(): RunnerScaleSetSession { + return this.session; + } + + async close(options: ScaleSetRequestOptions = {}): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/sessions/${this.currentSession.sessionId}`; + await this.actionsRequest('DELETE', path, { + expectedStatuses: [204], + signal: options.signal, + }); + } + + /** + * Long-poll for a batched scale set message. A 202 response means no message + * is currently available and is represented as `null`. + */ + async getMessage( + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.withMessageTokenRefresh( + (session) => this.getMessageWithSession(session, lastMessageId, maxCapacity, options), + options, + ); + } + + async pollMessage( + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions = {}, + ): Promise { + return this.getMessage(lastMessageId, maxCapacity, options); + } + + /** Delete a queue message after processing it, which acknowledges the batch. */ + async deleteMessage(messageId: number, options: ScaleSetRequestOptions = {}): Promise { + await this.withMessageTokenRefresh( + (session) => this.deleteMessageWithSession(session, messageId, options), + options, + ); + } + + async acknowledgeMessage(messageId: number, options: ScaleSetRequestOptions = {}): Promise { + return this.deleteMessage(messageId, options); + } + + /** Return the authoritative subset of runner request IDs acquired by the service. */ + async acquireJobs(requestIds: number[], options: ScaleSetRequestOptions = {}): Promise { + return this.withMessageTokenRefresh( + (session) => this.acquireJobsWithSession(session, requestIds, options), + options, + ); + } + + private async getMessageWithSession( + session: RunnerScaleSetSession, + lastMessageId: number, + maxCapacity: number, + options: ScaleSetRequestOptions, + ): Promise { + const url = new URL(session.messageQueueUrl); + if (lastMessageId > 0) { + url.searchParams.set('lastMessageId', String(lastMessageId)); + } + const result = await executeRequest( + this.fetchImplementation, + url, + { + method: 'GET', + headers: { + Accept: 'application/json; api-version=6.0-preview', + Authorization: `Bearer ${session.messageQueueAccessToken}`, + 'User-Agent': this.userAgent(), + [HEADER_SCALE_SET_MAX_CAPACITY]: String(maxCapacity), + }, + signal: options.signal, + }, + [200, 202], + (response) => (response.status === 401 ? SCALE_SET_ERROR_CODES.messageQueueTokenExpired : undefined), + ); + + if (result.response.status === 202) { + return null; + } + return parseRunnerScaleSetMessage(result, url); + } + + private async deleteMessageWithSession( + session: RunnerScaleSetSession, + messageId: number, + options: ScaleSetRequestOptions, + ): Promise { + const url = new URL(session.messageQueueUrl); + const valueAfterOrigin = session.messageQueueUrl.slice(url.origin.length); + const originalPath = valueAfterOrigin.startsWith('/') ? url.pathname : ''; + url.pathname = `${originalPath}/${messageId}`; + await executeRequest( + this.fetchImplementation, + url, + { + method: 'DELETE', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${session.messageQueueAccessToken}`, + 'User-Agent': this.userAgent(), + }, + signal: options.signal, + }, + [204], + (response) => (response.status === 401 ? SCALE_SET_ERROR_CODES.messageQueueTokenExpired : undefined), + ); + } + + private async acquireJobsWithSession( + session: RunnerScaleSetSession, + requestIds: number[], + options: ScaleSetRequestOptions, + ): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/acquirejobs`; + try { + const { result, url } = await this.actionsRequest('POST', path, { + body: requestIds, + expectedStatuses: [200], + authorization: `Bearer ${session.messageQueueAccessToken}`, + signal: options.signal, + }); + return parseJsonResponse(result, 'POST', url).value; + } catch (error) { + if (error instanceof ScaleSetHttpError && error.status === 401) { + throw new ScaleSetHttpError({ + method: error.method, + url: error.url, + status: error.status, + statusText: error.statusText, + headers: new Headers({ + ...(error.activityId ? { ActivityId: error.activityId } : {}), + ...(error.githubRequestId ? { 'X-GitHub-Request-Id': error.githubRequestId } : {}), + }), + responseBody: error.responseBody, + code: SCALE_SET_ERROR_CODES.messageQueueTokenExpired, + cause: error, + }); + } + throw error; + } + } + + private async withMessageTokenRefresh( + operation: (session: RunnerScaleSetSession) => Promise, + options: ScaleSetRequestOptions, + ): Promise { + const expiredSession = this.currentSession; + try { + return await operation(expiredSession); + } catch (error) { + if (!(error instanceof ScaleSetHttpError) || error.code !== SCALE_SET_ERROR_CODES.messageQueueTokenExpired) { + throw error; + } + } + + await this.refreshMessageSession(expiredSession, options); + return operation(this.currentSession); + } + + private async refreshMessageSession( + expiredSession: RunnerScaleSetSession, + options: ScaleSetRequestOptions, + ): Promise { + if ( + this.currentSession.sessionId !== expiredSession.sessionId || + this.currentSession.messageQueueAccessToken !== expiredSession.messageQueueAccessToken + ) { + return; + } + + if (this.sessionRefresh === undefined) { + this.sessionRefresh = this.doRefreshMessageSession(options).finally(() => { + this.sessionRefresh = undefined; + }); + } + await this.sessionRefresh; + } + + private async doRefreshMessageSession(options: ScaleSetRequestOptions): Promise { + const path = `/${SCALE_SET_ENDPOINT}/${this.runnerScaleSetId}/sessions/${this.currentSession.sessionId}`; + const { result, url } = await this.actionsRequest('PATCH', path, { + expectedStatuses: [200], + signal: options.signal, + }); + this.currentSession = validateSession( + normalizeSession(parseJsonResponse(result, 'PATCH', url)), + ); + } +} diff --git a/lambdas/libs/github-actions-scale-set/src/types.ts b/lambdas/libs/github-actions-scale-set/src/types.ts new file mode 100644 index 0000000000..4d800646ec --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/types.ts @@ -0,0 +1,198 @@ +export const DEFAULT_RUNNER_GROUP = 'default'; + +export const MESSAGE_TYPES = { + jobAvailable: 'JobAvailable', + jobAssigned: 'JobAssigned', + jobStarted: 'JobStarted', + jobCompleted: 'JobCompleted', +} as const; + +export type MessageType = (typeof MESSAGE_TYPES)[keyof typeof MESSAGE_TYPES]; + +export interface JobMessageBase { + messageType: MessageType; + runnerRequestId: number; + repositoryName: string; + ownerName: string; + jobId: string; + jobWorkflowRef: string; + jobDisplayName: string; + workflowRunId: number; + eventName: string; + requestLabels: string[]; + queueTime: string; + scaleSetAssignTime: string; + runnerAssignTime: string; + finishTime: string; +} + +export interface JobAvailable extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobAvailable; + acquireJobUrl: string; +} + +export interface JobAssigned extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobAssigned; +} + +export interface JobStarted extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobStarted; + runnerId: number; + runnerName: string; +} + +export interface JobCompleted extends JobMessageBase { + messageType: typeof MESSAGE_TYPES.jobCompleted; + result: string; + runnerId: number; + runnerName: string; +} + +export interface Label { + type?: string; + name: string; +} + +export interface RunnerGroup { + id: number; + name: string; + size: number; + isDefaultGroup: boolean; +} + +export interface RunnerSetting { + disableUpdate?: boolean; +} + +/** + * Runner scale set representation used by the Actions service. + * + * The same shape is accepted for create and update operations, so server-owned + * fields are optional. The client translates `runnerSetting` to the upstream + * wire key `RunnerSetting` when it sends a request. + */ +export interface RunnerScaleSet { + id?: number; + name?: string; + runnerGroupId?: number; + runnerGroupName?: string; + labels?: Label[]; + runnerSetting?: RunnerSetting; + createdOn?: string; + runnerJitConfigUrl?: string; + statistics?: RunnerScaleSetStatistic | null; +} + +export interface RunnerScaleSetJitRunnerSetting { + name: string; + workFolder?: string; +} + +export interface RunnerReference { + id: number; + name: string; + runnerScaleSetId: number; +} + +export interface GitHubRunnerReference { + id: number; + name: string; + status: 'online' | 'offline' | string; + busy: boolean; +} + +export interface ScaleSetRunnerState { + runnerId: number; + runnerName: string; + scaleSetId: number; + status: 'online' | 'offline' | 'unknown'; + busy: boolean | undefined; +} + +export interface RunnerScaleSetJitRunnerConfig { + runner: RunnerReference | null; + encodedJITConfig: string; +} + +export interface RunnerScaleSetStatistic { + totalAvailableJobs: number; + totalAcquiredJobs: number; + totalAssignedJobs: number; + totalRunningJobs: number; + totalRegisteredRunners: number; + totalBusyRunners: number; + totalIdleRunners: number; +} + +export interface RunnerScaleSetSession { + sessionId: string; + ownerName: string; + runnerScaleSet?: RunnerScaleSet | null; + messageQueueUrl: string; + messageQueueAccessToken: string; + statistics?: RunnerScaleSetStatistic | null; +} + +export interface RunnerScaleSetMessage { + messageId: number; + statistics: RunnerScaleSetStatistic | null; + jobAvailableMessages: JobAvailable[]; + jobAssignedMessages: JobAssigned[]; + jobStartedMessages: JobStarted[]; + jobCompletedMessages: JobCompleted[]; +} + +export interface SystemInfo { + system?: string; + version?: string; + commitSha?: string; + scaleSetId?: number; + subsystem?: string; +} + +export interface AccessToken { + token: string; + expiresAt?: string | Date; +} + +export type AccessTokenProvider = () => Promise; + +export type ScaleSetFetch = typeof globalThis.fetch; + +export interface ScaleSetRequestOptions { + signal?: AbortSignal; +} + +export interface ScaleSetRetryOptions { + /** Number of retries after the initial request for retry-eligible operations. */ + maxRetries?: number; + /** Initial exponential-backoff delay. */ + initialBackoffMs?: number; + /** Upper bound for exponential backoff and Retry-After delays. */ + maxBackoffMs?: number; + /** Timeout applied independently to each fetch attempt. */ + requestTimeoutMs?: number; +} + +interface ScaleSetClientBaseOptions { + gitHubConfigUrl: string; + systemInfo?: SystemInfo; + fetch?: ScaleSetFetch; + forceGhes?: boolean; + userAgent?: string; + /** Intended for deterministic tests. Defaults to `new Date()`. */ + now?: () => Date; + retry?: ScaleSetRetryOptions; +} + +export type GitHubActionsScaleSetClientOptions = ScaleSetClientBaseOptions & + ( + | { + personalAccessToken: string; + accessTokenProvider?: never; + } + | { + personalAccessToken?: never; + accessTokenProvider: AccessTokenProvider; + } + ); diff --git a/lambdas/libs/github-actions-scale-set/tsconfig.json b/lambdas/libs/github-actions-scale-set/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/libs/github-actions-scale-set/vitest.config.ts b/lambdas/libs/github-actions-scale-set/vitest.config.ts new file mode 100644 index 0000000000..c52b8d7522 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/vitest.config.ts @@ -0,0 +1,22 @@ +import { mergeConfig } from 'vitest/config'; + +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + root: __dirname, + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/**/*.d.ts', 'src/index.ts'], + thresholds: { + // Measured by the package-scoped wire-contract suite. These floors keep + // meaningful regression protection without pretending every defensive + // parser/error branch is exercised by production-path tests. + statements: 75, + branches: 60, + functions: 80, + lines: 75, + }, + }, + }, +}); diff --git a/lambdas/package.json b/lambdas/package.json index c6fa5d72c3..0a0b0b088a 100644 --- a/lambdas/package.json +++ b/lambdas/package.json @@ -3,7 +3,8 @@ "private": true, "workspaces": [ "functions/*", - "libs/*" + "libs/*", + "services/*" ], "scripts": { "build": "nx run-many --target=build --all", diff --git a/lambdas/services/scale-set/Dockerfile b/lambdas/services/scale-set/Dockerfile new file mode 100644 index 0000000000..d1bfbe429f --- /dev/null +++ b/lambdas/services/scale-set/Dockerfile @@ -0,0 +1,25 @@ +FROM node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 AS build +WORKDIR /workspace/lambdas + +COPY lambdas/package.json lambdas/yarn.lock lambdas/.yarnrc.yml ./ +COPY lambdas/.yarn ./.yarn +COPY lambdas/tsconfig.json lambdas/vitest.base.config.ts ./ +COPY lambdas/functions ./functions +COPY lambdas/libs ./libs +COPY lambdas/services ./services + +RUN corepack enable && yarn install --immutable +RUN yarn workspace @aws-github-runner/scale-set-service build + +FROM node:24-bookworm-slim@sha256:3638d9a6fe4030bd716be989438248074489337ba3275657f93595428be4fc03 AS runtime +ENV NODE_ENV=production \ + SCALE_SET_HEALTH_PORT=8080 +WORKDIR /app + +COPY --from=build --chown=node:node /workspace/lambdas/services/scale-set/dist/ ./ +COPY --from=build --chown=node:node /workspace/lambdas/services/scale-set/healthcheck.cjs ./healthcheck.cjs + +USER node +EXPOSE 8080 +HEALTHCHECK --interval=30s --timeout=5s --start-period=90s --retries=3 CMD ["node", "/app/healthcheck.cjs"] +ENTRYPOINT ["node", "--disable-proto=delete", "/app/index.js"] diff --git a/lambdas/services/scale-set/Dockerfile.dockerignore b/lambdas/services/scale-set/Dockerfile.dockerignore new file mode 100644 index 0000000000..7c72e0e7ab --- /dev/null +++ b/lambdas/services/scale-set/Dockerfile.dockerignore @@ -0,0 +1,7 @@ +**/coverage +**/dist +**/node_modules +.git +.nx +lambdas/.yarn/install-state.gz +*.log diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md new file mode 100644 index 0000000000..7bc3a46e01 --- /dev/null +++ b/lambdas/services/scale-set/README.md @@ -0,0 +1,108 @@ +# Scale-set controller service + +This workspace builds the long-running, topology-neutral controller used by the scale-set orchestration provider. + +Each controller group maps to one ECS service, one task definition, and normally one running task. The task contains one application container and one `ScaleSetController`, which supervises one independent reconciler per runner config: + +```text +ECS service (controller group) +└── one ECS task + └── one scale-set container + └── ScaleSetController + ├── reconciler: runner config A → scale set A → session A + └── reconciler: runner config B → scale set B → session B +``` + +A group is only a packing and deployment boundary. Every reconciler retains its own GitHub message session, lifecycle state, retry loop, health, and compute-provider instance. One reconciler failure does not exit the others. + +## Production configuration + +The ECS task receives only these group selectors: + +- `SCALE_SET_CONTROLLER_GROUP_NAME` +- `SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH` +- `SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION` + +The service reads every direct child under the SSM path with paginated `GetParametersByPath`. Each child name must equal its `runnerConfigName`, and each value uses this flat, versioned schema: + +```json +{ + "schemaVersion": 1, + "runnerConfigName": "linux-x64", + "githubConfigUrl": "https://github.com/example", + "scaleSetId": 123, + "expectedScaleSetName": "linux-x64", + "expectedRunnerGroupId": null, + "minRunners": 0, + "maxRunners": 20, + "bootTimeoutMinutes": 10, + "sslVerify": true, + "githubApp": { + "appIdParameterName": "/runners/github-app/id", + "privateKeyParameterName": "/runners/github-app/key", + "installationIdParameterName": "/runners/github-app/installation-id" + }, + "computeProvider": { + "type": "ec2", + "configuration": { + "region": "eu-west-1", + "environment": "example-linux-x64", + "runnerNamePrefix": "", + "jitConfigParameterPath": "/runners/example-linux-x64/tokens", + "subnets": ["subnet-0123456789abcdef0"], + "launchTemplateName": "example-linux-x64-action-runner", + "ec2instanceCriteria": { + "instanceTypes": ["m7i.large"], + "targetCapacityType": "spot", + "instanceAllocationStrategy": "price-capacity-optimized" + }, + "onDemandFailoverOnError": [], + "scaleErrors": [], + "useDedicatedHost": false, + "ssmParameterTags": [] + } + } +} +``` + +Optional fields are `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. `expectedRunnerGroupId` can be omitted or null. Before opening a session, the reconciler fetches the configured scale-set ID and verifies its expected name and, when supplied, runner-group ID. + +GitHub App values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. + +`SCALE_SET_CONTROLLER_MANIFEST` is supported only as a bounded local/test convenience. It contains `{ "version": 1, "groupName": "...", "reconcilers": [...] }` and uses the same reconciler objects. + +Runtime settings: + +| Environment variable | Default | +| --------------------------------------------- | ------- | +| `SCALE_SET_HEALTH_PORT` | `8080` | +| `SCALE_SET_HEALTH_STALE_AFTER_SECONDS` | `180` | +| `SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS` | `110` | +| `SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS` | `10` | +| `SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS` | `1` | +| `SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS` | `30` | + +## Reconciliation and health + +Demand is calculated as `max(totalAssignedJobs, min(maxRunners, minRunners + totalAssignedJobs))`. The maximum therefore bounds requested idle capacity without ever requesting scale-down below work GitHub has already assigned. Job-started and job-completed messages maintain a bounded in-memory lifecycle cache. State is unknown after restart unless GitHub provides an exact runner match, and the compute provider must retain unknown or busy runners. + +The public GitHub runner inventory is not fetched on ordinary steady-state or scale-up polls. A compute provider explicitly requests one bounded, owner-scope inventory refresh when it needs to verify old handed-off capacity or perform safe scale-down; owner inventory is briefly shared across reconcilers. The first provider pass is marked lifecycle-only and the second is explicitly marked inventory-complete, so the provider cannot mistake a post-restart gap for an authoritative absence. Runner deletion executes inside the serialized reconcile loop, re-fetches the Actions identity by name, and then performs a fresh public GitHub lookup to verify the exact ID/name and confirm the runner is not busy before issuing the delete. + +Message acknowledgement is last: acquire available jobs, update lifecycle state, complete the idempotent compute reconciliation, then delete the message. A failure leaves the message available for redelivery. A typed busy/unknown retention is processed and acknowledged without closing the session; it is not treated as an API failure. + +The EC2 provider counts a `config-published` instance as serving only during the orchestration request's boot window (`bootTimeoutMinutes`, default `10`) or after an exact online or `JobStarted` identity is observed. After the window, offline or unknown capacity is retained rather than terminated, and the complete inventory pass allows it to stop suppressing a replacement. Instances left in an earlier or unknown publication state are also retained for operator recovery and never terminated speculatively. EC2 ownership includes a SHA-256 hash of the canonical GitHub configuration scope, preventing the same runner-config name and numeric scale-set ID in another GitHub scope from colliding. A bounded one-instance physical surge may replace ambiguous capacity; once that ceiling is reached, the provider reports retained capacity instead of creating an unbounded replacement loop. + +- `GET /healthz` reports controller liveness and is used by Docker/ECS. External GitHub outages remain live but degraded to avoid restart loops. +- `GET /readyz` reports readiness and returns 503 unless every reconciler is ready. + +## Container + +Build from the repository root: + +```shell +docker build --target runtime -f lambdas/services/scale-set/Dockerfile -t scale-set-controller . +``` + +The image supports `linux/amd64` and `linux/arm64`, uses a digest-pinned multi-stage Node image, runs as the unprivileged `node` user, includes a Node-based health check, and does not require filesystem writes. Deploy with a read-only root filesystem, all Linux capabilities dropped, no Docker socket, and only the task-role permissions required by the selected group. + +The module's official GHCR package must allow anonymous pulls so the default image works without registry credentials. Production deployments should select a released image by digest and verify its provenance/attestation. A private ECR override requires `container.ecr_repository.arn`; private non-ECR registry credentials are not currently exposed by the Terraform orchestration module. diff --git a/lambdas/services/scale-set/healthcheck.cjs b/lambdas/services/scale-set/healthcheck.cjs new file mode 100644 index 0000000000..66c833829c --- /dev/null +++ b/lambdas/services/scale-set/healthcheck.cjs @@ -0,0 +1,13 @@ +'use strict'; + +const http = require('node:http'); +const port = Number(process.env.SCALE_SET_HEALTH_PORT || '8080'); +const request = http.get( + { host: '127.0.0.1', port, path: '/healthz', timeout: 4000, headers: { Connection: 'close' } }, + (response) => { + response.resume(); + process.exit(response.statusCode === 200 ? 0 : 1); + }, +); +request.on('timeout', () => request.destroy(new Error('health check timed out'))); +request.on('error', () => process.exit(1)); diff --git a/lambdas/services/scale-set/package.json b/lambdas/services/scale-set/package.json new file mode 100644 index 0000000000..9a912211bd --- /dev/null +++ b/lambdas/services/scale-set/package.json @@ -0,0 +1,42 @@ +{ + "name": "@aws-github-runner/scale-set-service", + "version": "0.1.0", + "private": true, + "main": "dist/index.js", + "type": "module", + "license": "MIT", + "scripts": { + "build": "ncc build src/main.ts -o dist --minify", + "typecheck": "tsc --noEmit", + "test": "NODE_ENV=test nx test", + "test:watch": "NODE_ENV=test nx test --watch", + "lint": "eslint src", + "format": "prettier --write \"**/*.ts\"", + "format-check": "prettier --check \"**/*.ts\"", + "all": "yarn typecheck && yarn build && yarn format-check && yarn lint && yarn test" + }, + "devDependencies": { + "@types/node": "^22.19.3", + "@vercel/ncc": "0.38.4", + "typescript": "^5.9.3" + }, + "dependencies": { + "@aws-github-runner/aws-ssm-util": "*", + "@aws-github-runner/compute-providers": "*", + "@aws-github-runner/github-actions-scale-set": "*", + "@aws-sdk/client-ssm": "^3.1009.0", + "@octokit/auth-app": "8.2.0", + "@octokit/request": "^9.2.2", + "undici": "^6.19.2" + }, + "nx": { + "includedScripts": [ + "build", + "typecheck", + "format", + "format-check", + "lint", + "all" + ] + } +} diff --git a/lambdas/services/scale-set/src/config.test.ts b/lambdas/services/scale-set/src/config.test.ts new file mode 100644 index 0000000000..6dfef9c445 --- /dev/null +++ b/lambdas/services/scale-set/src/config.test.ts @@ -0,0 +1,184 @@ +import { + MAX_MANIFEST_BYTES, + parseScaleSetControllerManifest, + parseScaleSetReconcilerConfig, + parseScaleSetServiceConfig, +} from './config'; + +function runnerConfig(overrides: Record = {}) { + return { + schemaVersion: 1, + runnerConfigName: 'linux-x64', + githubConfigUrl: 'https://github.com/example', + scaleSetId: 123, + expectedScaleSetName: 'linux-x64', + expectedRunnerGroupId: null, + minRunners: 0, + maxRunners: 20, + githubApp: { + appIdParameterName: '/runner/app/id', + privateKeyParameterName: '/runner/app/key', + installationIdParameterName: '/runner/app/installation-id', + }, + computeProvider: { type: 'ec2', configuration: { subnetIds: ['subnet-1'] } }, + ...overrides, + }; +} + +describe('scale-set service configuration', () => { + it('parses the production SSM group source and runtime defaults', () => { + expect( + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'ec2-default', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/runner/groups/ec2-default', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '42', + }), + ).toEqual({ + groupName: 'ec2-default', + groupConfigPath: '/runner/groups/ec2-default', + groupRevision: '42', + healthPort: 8080, + healthStaleAfterMs: 180000, + shutdownTimeoutMs: 110000, + sessionCloseTimeoutMs: 10000, + reconnectInitialBackoffMs: 1000, + reconnectMaxBackoffMs: 30000, + }); + }); + + it('supports bounded inline manifests for local use', () => { + const manifest = JSON.stringify({ version: 1, groupName: 'local', reconcilers: [runnerConfig()] }); + expect(parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: manifest }).manifest).toBe(manifest); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_MANIFEST: manifest, + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/both', + }), + ).toThrow('provide exactly one'); + expect(() => + parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: 'x'.repeat(MAX_MANIFEST_BYTES + 1) }), + ).toThrow('must not exceed'); + }); + + it('validates production selectors and numeric runtime settings', () => { + expect(() => parseScaleSetServiceConfig({})).toThrow('provide exactly one'); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'bad name', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '1', + }), + ).toThrow('group name'); + expect(() => + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_GROUP_NAME: 'valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH: '/valid', + SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION: '1', + SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS: '31', + SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS: '30', + }), + ).toThrow('must not exceed'); + }); +}); + +describe('parseScaleSetControllerManifest', () => { + it('parses the frozen flat runner-config schema and defaults', () => { + expect(parseScaleSetReconcilerConfig(runnerConfig(), 0, 'group')).toMatchObject({ + schemaVersion: 1, + runnerConfigName: 'linux-x64', + expectedScaleSetName: 'linux-x64', + bootTimeoutMinutes: 10, + sessionOwner: 'group.linux-x64', + workFolder: '_work', + forceGhes: false, + sslVerify: true, + computeProvider: { type: 'ec2', configuration: { subnetIds: ['subnet-1'] } }, + }); + }); + + it('normalizes explicit optional settings', () => { + expect( + parseScaleSetReconcilerConfig( + runnerConfig({ + expectedRunnerGroupId: 7, + sessionOwner: 'owner/group', + workFolder: 'runner/_work', + forceGhes: true, + sslVerify: false, + userAgent: 'github-aws-runners/test', + bootTimeoutMinutes: 30, + }), + 0, + 'group', + ), + ).toMatchObject({ + expectedRunnerGroupId: 7, + sessionOwner: 'owner/group', + workFolder: 'runner/_work', + forceGhes: true, + sslVerify: false, + bootTimeoutMinutes: 30, + }); + }); + + it('bounds the derived session owner for maximum-length names', () => { + const parsed = parseScaleSetReconcilerConfig( + runnerConfig({ runnerConfigName: 'r'.repeat(128) }), + 0, + 'g'.repeat(128), + ); + expect(parsed.sessionOwner).toHaveLength(256); + expect(parsed.sessionOwner).toMatch(/\.[a-f0-9]{16}$/); + }); + + it('rejects unsafe URLs, unknown fields, prototype keys, and schema drift', () => { + expect(() => + parseScaleSetReconcilerConfig(runnerConfig({ githubConfigUrl: 'http://github.com/example' }), 0, 'g'), + ).toThrow('must use HTTPS'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ extra: true }), 0, 'g')).toThrow('unknown field'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ schemaVersion: 2 }), 0, 'g')).toThrow('schemaVersion'); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ bootTimeoutMinutes: 0 }), 0, 'g')).toThrow( + 'bootTimeoutMinutes', + ); + expect(() => parseScaleSetReconcilerConfig(runnerConfig({ bootTimeoutMinutes: 121 }), 0, 'g')).toThrow( + 'bootTimeoutMinutes', + ); + const polluted = JSON.parse('{"__proto__":{"admin":true}}') as unknown; + expect(() => + parseScaleSetReconcilerConfig( + runnerConfig({ computeProvider: { type: 'ec2', configuration: polluted } }), + 0, + 'g', + ), + ).toThrow('forbidden field'); + }); + + it('rejects duplicate runner names and scale-set IDs within an equivalent GitHub scope', () => { + expect(() => + parseScaleSetControllerManifest({ version: 1, groupName: 'g', reconcilers: [runnerConfig(), runnerConfig()] }), + ).toThrow('duplicated'); + expect(() => + parseScaleSetControllerManifest({ + version: 1, + groupName: 'g', + reconcilers: [ + runnerConfig(), + runnerConfig({ runnerConfigName: 'other', githubConfigUrl: 'https://GITHUB.com/example/' }), + ], + }), + ).toThrow('scale set ID'); + }); + + it('allows the same numeric scale-set ID in different GitHub scopes', () => { + expect( + parseScaleSetControllerManifest({ + version: 1, + groupName: 'g', + reconcilers: [ + runnerConfig(), + runnerConfig({ runnerConfigName: 'other', githubConfigUrl: 'https://github.com/another' }), + ], + }).reconcilers, + ).toHaveLength(2); + }); +}); diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts new file mode 100644 index 0000000000..15e98df45f --- /dev/null +++ b/lambdas/services/scale-set/src/config.ts @@ -0,0 +1,479 @@ +import { createHash } from 'node:crypto'; + +export const SCALE_SET_CONTROLLER_MANIFEST_VERSION = 1; +export const MAX_MANIFEST_BYTES = 256 * 1024; + +export type JsonPrimitive = boolean | number | string | null; +export type JsonValue = JsonPrimitive | JsonValue[] | { readonly [key: string]: JsonValue }; + +export interface GitHubAppParameterReferences { + appIdParameterName: string; + installationIdParameterName: string; + privateKeyParameterName: string; +} + +export interface ScaleSetReconcilerConfig { + schemaVersion: 1; + runnerConfigName: string; + scaleSetId: number; + expectedScaleSetName: string; + expectedRunnerGroupId?: number; + githubConfigUrl: string; + githubApp: GitHubAppParameterReferences; + computeProvider: { + type: string; + configuration: Readonly>; + }; + minRunners: number; + maxRunners: number; + bootTimeoutMinutes: number; + sessionOwner: string; + workFolder: string; + forceGhes: boolean; + sslVerify: boolean; + userAgent?: string; +} + +export interface ScaleSetControllerManifest { + version: typeof SCALE_SET_CONTROLLER_MANIFEST_VERSION; + groupName: string; + revision?: string; + reconcilers: readonly ScaleSetReconcilerConfig[]; +} + +export interface ScaleSetServiceConfig { + manifest?: string; + groupConfigPath?: string; + groupName?: string; + groupRevision?: string; + healthPort: number; + healthStaleAfterMs: number; + shutdownTimeoutMs: number; + sessionCloseTimeoutMs: number; + reconnectInitialBackoffMs: number; + reconnectMaxBackoffMs: number; +} + +export type ScaleSetServiceEnvironment = Readonly>; + +const MAX_SCALE_SET_CAPACITY = 2_147_483_647; +const DEFAULT_BOOT_TIMEOUT_MINUTES = 10; +const MAX_BOOT_TIMEOUT_MINUTES = 120; +const DEFAULT_HEALTH_PORT = 8080; +const DEFAULT_HEALTH_STALE_AFTER_SECONDS = 180; +const DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 110; +const DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS = 10; +const DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS = 1; +const DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS = 30; +const MAX_RECONCILERS = 1000; +const MAX_PROVIDER_CONFIG_NODES = 10_000; +const MAX_PROVIDER_CONFIG_DEPTH = 32; +const SAFE_NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const SAFE_PROVIDER_TYPE = /^[a-z][a-z0-9_-]{0,63}$/; +const SAFE_SSM_PARAMETER = /^\/[A-Za-z0-9_.\-/]{1,2047}$/; +const PROTOTYPE_KEYS = new Set(['__proto__', 'constructor', 'prototype']); + +export class ScaleSetConfigurationError extends Error { + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'ScaleSetConfigurationError'; + } +} + +function parseInteger( + environment: ScaleSetServiceEnvironment, + name: string, + options: { defaultValue: number; minimum: number; maximum: number }, +): number { + const raw = environment[name]?.trim(); + if (!raw) return options.defaultValue; + if (!/^\d+$/.test(raw)) throw new ScaleSetConfigurationError(`${name} must be an integer`); + const value = Number(raw); + if (!Number.isSafeInteger(value) || value < options.minimum || value > options.maximum) { + throw new ScaleSetConfigurationError(`${name} must be between ${options.minimum} and ${options.maximum}`); + } + return value; +} + +export function parseScaleSetServiceConfig(environment: ScaleSetServiceEnvironment): ScaleSetServiceConfig { + const manifest = environment.SCALE_SET_CONTROLLER_MANIFEST?.trim(); + const groupConfigPath = environment.SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH?.trim(); + if ((manifest === undefined || manifest === '') === (groupConfigPath === undefined || groupConfigPath === '')) { + throw new ScaleSetConfigurationError( + 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + ); + } + if (manifest !== undefined && Buffer.byteLength(manifest, 'utf8') > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError(`SCALE_SET_CONTROLLER_MANIFEST must not exceed ${MAX_MANIFEST_BYTES} bytes`); + } + let groupName: string | undefined; + let groupRevision: string | undefined; + if (groupConfigPath !== undefined) { + validateSsmParameterName(groupConfigPath, 'group config path'); + groupName = validateSafeName(environment.SCALE_SET_CONTROLLER_GROUP_NAME?.trim() ?? '', 'group name'); + groupRevision = environment.SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION?.trim(); + if (!groupRevision || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(groupRevision)) { + throw new ScaleSetConfigurationError('SCALE_SET_CONTROLLER_GROUP_CONFIG_REVISION is invalid'); + } + } + + const reconnectInitialBackoffMs = + parseInteger(environment, 'SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS', { + defaultValue: DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS, + minimum: 1, + maximum: 300, + }) * 1000; + const reconnectMaxBackoffMs = + parseInteger(environment, 'SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS', { + defaultValue: DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS, + minimum: 1, + maximum: 3600, + }) * 1000; + if (reconnectInitialBackoffMs > reconnectMaxBackoffMs) { + throw new ScaleSetConfigurationError( + 'SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS must not exceed SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS', + ); + } + + return { + ...(manifest ? { manifest } : {}), + ...(groupConfigPath ? { groupConfigPath, groupName, groupRevision } : {}), + healthPort: parseInteger(environment, 'SCALE_SET_HEALTH_PORT', { + defaultValue: DEFAULT_HEALTH_PORT, + minimum: 1, + maximum: 65535, + }), + healthStaleAfterMs: + parseInteger(environment, 'SCALE_SET_HEALTH_STALE_AFTER_SECONDS', { + defaultValue: DEFAULT_HEALTH_STALE_AFTER_SECONDS, + minimum: 30, + maximum: 3600, + }) * 1000, + shutdownTimeoutMs: + parseInteger(environment, 'SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS', { + defaultValue: DEFAULT_SHUTDOWN_TIMEOUT_SECONDS, + minimum: 1, + maximum: 300, + }) * 1000, + sessionCloseTimeoutMs: + parseInteger(environment, 'SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS', { + defaultValue: DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS, + minimum: 1, + maximum: 60, + }) * 1000, + reconnectInitialBackoffMs, + reconnectMaxBackoffMs, + }; +} + +function objectValue(value: unknown, path: string): Record { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ScaleSetConfigurationError(`${path} must be an object`); + } + return value as Record; +} + +function exactKeys(value: Record, allowed: readonly string[], path: string): void { + const allowedSet = new Set(allowed); + const unknown = Object.keys(value).filter((key) => !allowedSet.has(key)); + if (unknown.length > 0) + throw new ScaleSetConfigurationError(`${path} contains unknown field ${JSON.stringify(unknown[0])}`); +} + +function requiredString(value: Record, key: string, path: string): string { + const result = value[key]; + if (typeof result !== 'string' || result.trim() === '') { + throw new ScaleSetConfigurationError(`${path}.${key} must be a non-empty string`); + } + return result.trim(); +} + +function optionalString(value: Record, key: string, path: string): string | undefined { + const result = value[key]; + if (result === undefined) return undefined; + if (typeof result !== 'string' || result.trim() === '') { + throw new ScaleSetConfigurationError(`${path}.${key} must be a non-empty string when set`); + } + return result.trim(); +} + +function integer(value: Record, key: string, path: string, minimum: number, maximum: number): number { + const result = value[key]; + if (!Number.isSafeInteger(result) || (result as number) < minimum || (result as number) > maximum) { + throw new ScaleSetConfigurationError(`${path}.${key} must be an integer between ${minimum} and ${maximum}`); + } + return result as number; +} + +function optionalBoolean(value: Record, key: string, path: string, fallback: boolean): boolean { + const result = value[key]; + if (result === undefined) return fallback; + if (typeof result !== 'boolean') throw new ScaleSetConfigurationError(`${path}.${key} must be a boolean`); + return result; +} + +function validateSafeName(value: string, path: string): string { + if (!SAFE_NAME.test(value)) { + throw new ScaleSetConfigurationError( + `${path} must start with an ASCII letter or digit and contain only letters, digits, dots, underscores, or hyphens`, + ); + } + return value; +} + +function validateSsmParameterName(value: string, path: string): string { + if (!SAFE_SSM_PARAMETER.test(value) || value.includes('//') || value.endsWith('/')) { + throw new ScaleSetConfigurationError(`${path} must be an absolute SSM parameter name`); + } + return value; +} + +function validateGitHubConfigUrl(raw: string, path: string): string { + let url: URL; + try { + url = new URL(raw); + } catch (error) { + throw new ScaleSetConfigurationError(`${path} must be a valid URL`, { cause: error }); + } + if (url.protocol !== 'https:') throw new ScaleSetConfigurationError(`${path} must use HTTPS`); + if (url.username || url.password || url.search || url.hash) { + throw new ScaleSetConfigurationError(`${path} must not contain credentials, a query, or a fragment`); + } + const parts = url.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean); + if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { + throw new ScaleSetConfigurationError(`${path} must identify a GitHub organization, repository, or enterprise`); + } + url.pathname = `/${parts.join('/')}`; + return url.toString().replace(/\/$/, ''); +} + +function validateWorkFolder(value: string, path: string): string { + if ( + value.length > 128 || + value.startsWith('/') || + value.includes('\\') || + value.split('/').some((part) => part === '' || part === '.' || part === '..') || + !/^[A-Za-z0-9._/-]+$/.test(value) + ) { + throw new ScaleSetConfigurationError(`${path} must be a safe relative path`); + } + return value; +} + +function validateUserAgent(value: string | undefined, path: string): string | undefined { + if (value === undefined) return undefined; + if (value.length > 256 || !/^[\x20-\x7E]+$/.test(value)) { + throw new ScaleSetConfigurationError(`${path} must contain at most 256 visible ASCII characters`); + } + return value; +} + +function validateJsonValue(value: unknown, path: string, depth = 0, counter = { value: 0 }): JsonValue { + counter.value += 1; + if (counter.value > MAX_PROVIDER_CONFIG_NODES || depth > MAX_PROVIDER_CONFIG_DEPTH) { + throw new ScaleSetConfigurationError(`${path} exceeds the provider configuration complexity limit`); + } + if (value === null || typeof value === 'boolean' || typeof value === 'string') return value; + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new ScaleSetConfigurationError(`${path} contains a non-finite number`); + return value; + } + if (Array.isArray(value)) + return value.map((item, index) => validateJsonValue(item, `${path}[${index}]`, depth + 1, counter)); + const record = objectValue(value, path); + const result: Record = Object.create(null) as Record; + for (const [key, child] of Object.entries(record)) { + if (PROTOTYPE_KEYS.has(key)) + throw new ScaleSetConfigurationError(`${path} contains forbidden field ${JSON.stringify(key)}`); + if (key.length === 0 || key.length > 128) + throw new ScaleSetConfigurationError(`${path} contains an invalid field name`); + result[key] = validateJsonValue(child, `${path}.${key}`, depth + 1, counter); + } + return result; +} + +function parseGitHubApp(value: unknown, path: string): GitHubAppParameterReferences { + const record = objectValue(value, path); + exactKeys(record, ['appIdParameterName', 'installationIdParameterName', 'privateKeyParameterName'], path); + return { + appIdParameterName: validateSsmParameterName( + requiredString(record, 'appIdParameterName', path), + `${path}.appIdParameterName`, + ), + installationIdParameterName: validateSsmParameterName( + requiredString(record, 'installationIdParameterName', path), + `${path}.installationIdParameterName`, + ), + privateKeyParameterName: validateSsmParameterName( + requiredString(record, 'privateKeyParameterName', path), + `${path}.privateKeyParameterName`, + ), + }; +} + +function parseComputeProvider(value: unknown, path: string): ScaleSetReconcilerConfig['computeProvider'] { + const record = objectValue(value, path); + exactKeys(record, ['type', 'configuration'], path); + const type = requiredString(record, 'type', path); + if (!SAFE_PROVIDER_TYPE.test(type)) throw new ScaleSetConfigurationError(`${path}.type is invalid`); + const configuration = validateJsonValue(record.configuration, `${path}.configuration`); + if (typeof configuration !== 'object' || configuration === null || Array.isArray(configuration)) { + throw new ScaleSetConfigurationError(`${path}.configuration must be an object`); + } + return { type, configuration }; +} + +export function parseScaleSetReconcilerConfig( + value: unknown, + index: number, + groupName: string, + basePath = 'manifest.reconcilers', +): ScaleSetReconcilerConfig { + const path = `${basePath}[${index}]`; + const record = objectValue(value, path); + exactKeys( + record, + [ + 'schemaVersion', + 'runnerConfigName', + 'scaleSetId', + 'expectedScaleSetName', + 'expectedRunnerGroupId', + 'githubConfigUrl', + 'githubApp', + 'computeProvider', + 'minRunners', + 'maxRunners', + 'bootTimeoutMinutes', + 'sessionOwner', + 'workFolder', + 'forceGhes', + 'sslVerify', + 'userAgent', + ], + path, + ); + const runnerConfigName = validateSafeName( + requiredString(record, 'runnerConfigName', path), + `${path}.runnerConfigName`, + ); + const minRunners = integer(record, 'minRunners', path, 0, MAX_SCALE_SET_CAPACITY); + const maxRunners = integer(record, 'maxRunners', path, 0, MAX_SCALE_SET_CAPACITY); + const bootTimeoutMinutes = + record.bootTimeoutMinutes === undefined + ? DEFAULT_BOOT_TIMEOUT_MINUTES + : integer(record, 'bootTimeoutMinutes', path, 1, MAX_BOOT_TIMEOUT_MINUTES); + if (minRunners > maxRunners) throw new ScaleSetConfigurationError(`${path}.minRunners must not exceed maxRunners`); + const sessionOwner = optionalString(record, 'sessionOwner', path) ?? defaultSessionOwner(groupName, runnerConfigName); + if (!/^[A-Za-z0-9][A-Za-z0-9._:/-]{0,255}$/.test(sessionOwner)) { + throw new ScaleSetConfigurationError(`${path}.sessionOwner is invalid`); + } + const userAgent = validateUserAgent(optionalString(record, 'userAgent', path), `${path}.userAgent`); + if (record.schemaVersion !== 1) throw new ScaleSetConfigurationError(`${path}.schemaVersion must be 1`); + const expectedRunnerGroupId = + record.expectedRunnerGroupId === undefined || record.expectedRunnerGroupId === null + ? undefined + : integer(record, 'expectedRunnerGroupId', path, 1, MAX_SCALE_SET_CAPACITY); + return { + schemaVersion: 1, + runnerConfigName, + scaleSetId: integer(record, 'scaleSetId', path, 1, MAX_SCALE_SET_CAPACITY), + expectedScaleSetName: validateScaleSetName( + requiredString(record, 'expectedScaleSetName', path), + `${path}.expectedScaleSetName`, + ), + ...(expectedRunnerGroupId === undefined ? {} : { expectedRunnerGroupId }), + githubConfigUrl: validateGitHubConfigUrl( + requiredString(record, 'githubConfigUrl', path), + `${path}.githubConfigUrl`, + ), + githubApp: parseGitHubApp(record.githubApp, `${path}.githubApp`), + computeProvider: parseComputeProvider(record.computeProvider, `${path}.computeProvider`), + minRunners, + maxRunners, + bootTimeoutMinutes, + sessionOwner, + workFolder: validateWorkFolder(optionalString(record, 'workFolder', path) ?? '_work', `${path}.workFolder`), + forceGhes: optionalBoolean(record, 'forceGhes', path, false), + sslVerify: optionalBoolean(record, 'sslVerify', path, true), + ...(userAgent === undefined ? {} : { userAgent }), + }; +} + +function defaultSessionOwner(groupName: string, runnerConfigName: string): string { + const candidate = `${groupName}.${runnerConfigName}`; + if (candidate.length <= 256) return candidate; + const suffix = createHash('sha256').update(candidate).digest('hex').slice(0, 16); + return `${candidate.slice(0, 239)}.${suffix}`; +} + +function validateScaleSetName(value: string, path: string): string { + if (value.length > 128 || !/^[\x20-\x7E]+$/.test(value)) { + throw new ScaleSetConfigurationError(`${path} must contain at most 128 visible ASCII characters`); + } + return value; +} + +export function parseScaleSetControllerManifest(input: string | unknown): ScaleSetControllerManifest { + let parsed = input; + if (typeof input === 'string') { + if (Buffer.byteLength(input, 'utf8') > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError(`controller manifest must not exceed ${MAX_MANIFEST_BYTES} bytes`); + } + try { + parsed = JSON.parse(input) as unknown; + } catch (error) { + throw new ScaleSetConfigurationError('controller manifest must contain valid JSON', { cause: error }); + } + } + const manifest = objectValue(parsed, 'manifest'); + exactKeys(manifest, ['version', 'groupName', 'revision', 'reconcilers'], 'manifest'); + if (manifest.version !== SCALE_SET_CONTROLLER_MANIFEST_VERSION) { + throw new ScaleSetConfigurationError(`manifest.version must be ${SCALE_SET_CONTROLLER_MANIFEST_VERSION}`); + } + const groupName = validateSafeName(requiredString(manifest, 'groupName', 'manifest'), 'manifest.groupName'); + if ( + !Array.isArray(manifest.reconcilers) || + manifest.reconcilers.length < 1 || + manifest.reconcilers.length > MAX_RECONCILERS + ) { + throw new ScaleSetConfigurationError(`manifest.reconcilers must contain between 1 and ${MAX_RECONCILERS} entries`); + } + const revision = optionalString(manifest, 'revision', 'manifest'); + if (revision !== undefined && !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(revision)) { + throw new ScaleSetConfigurationError('manifest.revision is invalid'); + } + const reconcilers = manifest.reconcilers.map((value, index) => + parseScaleSetReconcilerConfig(value, index, groupName), + ); + validateUniqueReconcilers(reconcilers); + return { + version: SCALE_SET_CONTROLLER_MANIFEST_VERSION, + groupName, + ...(revision === undefined ? {} : { revision }), + reconcilers, + }; +} + +export function validateUniqueReconcilers(reconcilers: readonly ScaleSetReconcilerConfig[]): void { + const names = new Set(); + const scopedScaleSetIds = new Set(); + for (const reconciler of reconcilers) { + if (names.has(reconciler.runnerConfigName)) { + throw new ScaleSetConfigurationError( + `runner config ${JSON.stringify(reconciler.runnerConfigName)} is duplicated`, + ); + } + const scopedScaleSetId = `${reconciler.githubConfigUrl}\u0000${reconciler.scaleSetId}`; + if (scopedScaleSetIds.has(scopedScaleSetId)) { + throw new ScaleSetConfigurationError( + `scale set ID ${reconciler.scaleSetId} is duplicated within GitHub scope ${JSON.stringify(reconciler.githubConfigUrl)}`, + ); + } + names.add(reconciler.runnerConfigName); + scopedScaleSetIds.add(scopedScaleSetId); + } +} diff --git a/lambdas/services/scale-set/src/controller.ts b/lambdas/services/scale-set/src/controller.ts new file mode 100644 index 0000000000..f575f60dd2 --- /dev/null +++ b/lambdas/services/scale-set/src/controller.ts @@ -0,0 +1,47 @@ +import type { ScaleSetControllerManifest, ScaleSetServiceConfig } from './config'; +import { ScaleSetControllerHealth } from './health'; +import type { ScaleSetLogger } from './logger'; +import { ScaleSetReconciler, type ScaleSetReconcilerDependencies } from './reconciler'; + +export class ScaleSetController { + readonly health: ScaleSetControllerHealth; + + constructor( + private readonly manifest: ScaleSetControllerManifest, + private readonly serviceConfig: ScaleSetServiceConfig, + private readonly dependencies: ScaleSetReconcilerDependencies, + private readonly controllerLogger: ScaleSetLogger, + ) { + this.health = new ScaleSetControllerHealth( + manifest.groupName, + manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + serviceConfig.healthStaleAfterMs, + ); + } + + async run(signal: AbortSignal): Promise { + const completions = this.manifest.reconcilers.map(async (config) => { + const status = this.health.reporter(config.runnerConfigName); + try { + await new ScaleSetReconciler(config, this.serviceConfig, this.dependencies).run(signal, status); + } catch (error) { + status.markFailed(error); + this.controllerLogger.error('scale_set_reconciler_uncaught_failure', { + runnerConfigName: config.runnerConfigName, + scaleSetId: config.scaleSetId, + error, + }); + } + }); + + await Promise.race([Promise.all(completions), waitForAbort(signal)]); + if (!signal.aborted) await waitForAbort(signal); + this.health.markStopping(); + await Promise.all(completions); + } +} + +async function waitForAbort(signal: AbortSignal): Promise { + if (signal.aborted) return; + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); +} diff --git a/lambdas/services/scale-set/src/credentials.test.ts b/lambdas/services/scale-set/src/credentials.test.ts new file mode 100644 index 0000000000..3910bd24d7 --- /dev/null +++ b/lambdas/services/scale-set/src/credentials.test.ts @@ -0,0 +1,111 @@ +const authMocks = vi.hoisted(() => ({ createAppAuth: vi.fn(), requestDefaults: vi.fn() })); +vi.mock('@octokit/auth-app', () => ({ createAppAuth: authMocks.createAppAuth })); +vi.mock('@octokit/request', () => ({ request: { defaults: authMocks.requestDefaults } })); + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +import { createGitHubAppAccessTokenProvider, loadGitHubAppCredentials, type ParameterStore } from './credentials'; + +const references = { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', +}; + +function encodedKey(body: string): string { + return Buffer.from(`-----BEGIN PRIVATE KEY-----\n${body}\n-----END PRIVATE KEY-----\n`).toString('base64'); +} + +describe('GitHub App credentials', () => { + beforeEach(() => { + vi.clearAllMocks(); + authMocks.requestDefaults.mockReturnValue(vi.fn()); + }); + + it('validates and decodes referenced SSM values', async () => { + const store: ParameterStore = { + get: vi.fn().mockResolvedValue( + new Map([ + ['/app/id', '123'], + ['/app/installation', '456'], + ['/app/key', encodedKey('abc')], + ]), + ), + }; + await expect(loadGitHubAppCredentials(references, store)).resolves.toMatchObject({ + appId: '123', + installationId: 456, + privateKey: expect.stringContaining('BEGIN PRIVATE KEY'), + }); + }); + + it('reuses auth for unchanged credentials and recreates it after rotation', async () => { + const values = [encodedKey('first'), encodedKey('first'), encodedKey('second')]; + const store: ParameterStore = { + get: vi.fn( + async () => + new Map([ + ['/app/id', '123'], + ['/app/installation', '456'], + ['/app/key', values.shift() as string], + ]), + ), + }; + const firstAuth = vi.fn().mockResolvedValue({ token: 'token-one', expiresAt: '2099-01-01T00:00:00Z' }); + const secondAuth = vi.fn().mockResolvedValue({ token: 'token-two', expiresAt: '2099-01-01T00:00:00Z' }); + const fetchImplementation = vi.fn(); + authMocks.createAppAuth.mockReturnValueOnce(firstAuth).mockReturnValueOnce(secondAuth); + + const provider = await createGitHubAppAccessTokenProvider( + references, + 'https://github.com/example', + false, + store, + fetchImplementation, + ); + await provider(); + await provider(); + await provider(); + + expect(store.get).toHaveBeenCalledTimes(3); + expect(authMocks.createAppAuth).toHaveBeenCalledTimes(2); + expect(authMocks.requestDefaults).toHaveBeenCalledWith({ + baseUrl: 'https://api.github.com', + request: { fetch: fetchImplementation }, + }); + expect(firstAuth).toHaveBeenCalledTimes(2); + expect(secondAuth).toHaveBeenCalledTimes(1); + }); + + it.each([ + [new Map([['/app/id', '123']]), 'was not returned'], + [ + new Map([ + ['/app/id', 'bad id'], + ['/app/installation', '1'], + ['/app/key', encodedKey('abc')], + ]), + 'App ID', + ], + [ + new Map([ + ['/app/id', '1'], + ['/app/installation', 'zero'], + ['/app/key', encodedKey('abc')], + ]), + 'positive integer', + ], + [ + new Map([ + ['/app/id', '1'], + ['/app/installation', '2'], + ['/app/key', 'not-base64'], + ]), + 'canonical base64', + ], + ])('rejects malformed credential parameters', async (values, message) => { + await expect(loadGitHubAppCredentials(references, { get: vi.fn().mockResolvedValue(values) })).rejects.toThrow( + message, + ); + }); +}); diff --git a/lambdas/services/scale-set/src/credentials.ts b/lambdas/services/scale-set/src/credentials.ts new file mode 100644 index 0000000000..9e3a87be11 --- /dev/null +++ b/lambdas/services/scale-set/src/credentials.ts @@ -0,0 +1,125 @@ +import { createAppAuth } from '@octokit/auth-app'; +import { request } from '@octokit/request'; +import { createHash } from 'node:crypto'; + +import { + githubApiUrl, + parseGitHubConfigUrl, + type AccessToken, + type ScaleSetFetch, +} from '@aws-github-runner/github-actions-scale-set'; + +import { ScaleSetConfigurationError, type GitHubAppParameterReferences } from './config'; + +export interface ParameterStore { + get(names: readonly string[]): Promise>; +} + +interface GitHubAppCredentials { + appId: string; + installationId: number; + privateKey: string; +} + +const MAX_PRIVATE_KEY_BYTES = 64 * 1024; + +function requiredParameter(values: ReadonlyMap, name: string): string { + const value = values.get(name); + if (value === undefined || value === '') { + throw new ScaleSetConfigurationError(`required SSM parameter ${JSON.stringify(name)} was not returned`); + } + return value; +} + +function decodePrivateKey(encoded: string): string { + if ( + encoded.length === 0 || + encoded.length > Math.ceil((MAX_PRIVATE_KEY_BYTES * 4) / 3) + 4 || + encoded.length % 4 !== 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(encoded) + ) { + throw new ScaleSetConfigurationError('GitHub App private key parameter must contain canonical base64'); + } + const decoded = Buffer.from(encoded, 'base64').toString('utf8').replace(/\\n/g, '\n'); + if (Buffer.byteLength(decoded, 'utf8') > MAX_PRIVATE_KEY_BYTES) { + throw new ScaleSetConfigurationError('GitHub App private key is too large'); + } + if (!/^-----BEGIN (?:RSA )?PRIVATE KEY-----\n[\s\S]+\n-----END (?:RSA )?PRIVATE KEY-----\n?$/.test(decoded)) { + throw new ScaleSetConfigurationError('GitHub App private key parameter is not a supported PEM private key'); + } + return decoded; +} + +export async function loadGitHubAppCredentials( + references: GitHubAppParameterReferences, + parameterStore: ParameterStore, +): Promise { + const values = await parameterStore.get([ + references.appIdParameterName, + references.installationIdParameterName, + references.privateKeyParameterName, + ]); + const appId = requiredParameter(values, references.appIdParameterName).trim(); + if (!/^[A-Za-z0-9_-]{1,128}$/.test(appId)) { + throw new ScaleSetConfigurationError('GitHub App ID parameter is invalid'); + } + const installationIdRaw = requiredParameter(values, references.installationIdParameterName).trim(); + if (!/^\d+$/.test(installationIdRaw)) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + const installationId = Number(installationIdRaw); + if (!Number.isSafeInteger(installationId) || installationId <= 0) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + return { + appId, + installationId, + privateKey: decodePrivateKey(requiredParameter(values, references.privateKeyParameterName)), + }; +} + +export async function createGitHubAppAccessTokenProvider( + references: GitHubAppParameterReferences, + githubConfigUrl: string, + forceGhes: boolean, + parameterStore: ParameterStore, + fetchImplementation: ScaleSetFetch = globalThis.fetch, +): Promise<() => Promise> { + const parsedConfig = parseGitHubConfigUrl(githubConfigUrl, forceGhes); + const apiBaseUrl = githubApiUrl(parsedConfig, '/').toString().replace(/\/$/, ''); + let cached: + | { + fingerprint: string; + installationId: number; + auth: ReturnType; + } + | undefined; + + return async () => { + // Reload references for rotation visibility, but preserve the Octokit auth + // instance while credentials are unchanged so its installation-token cache + // remains effective. + const credentials = await loadGitHubAppCredentials(references, parameterStore); + const fingerprint = createHash('sha256') + .update(credentials.appId) + .update('\u0000') + .update(String(credentials.installationId)) + .update('\u0000') + .update(credentials.privateKey) + .digest('base64url'); + if (cached?.fingerprint !== fingerprint) { + cached = { + fingerprint, + installationId: credentials.installationId, + auth: createAppAuth({ + appId: credentials.appId, + installationId: credentials.installationId, + privateKey: credentials.privateKey, + request: request.defaults({ baseUrl: apiBaseUrl, request: { fetch: fetchImplementation } }), + }), + }; + } + const installation = await cached.auth({ type: 'installation', installationId: cached.installationId }); + return { token: installation.token, expiresAt: installation.expiresAt }; + }; +} diff --git a/lambdas/services/scale-set/src/github-http.test.ts b/lambdas/services/scale-set/src/github-http.test.ts new file mode 100644 index 0000000000..870399ac84 --- /dev/null +++ b/lambdas/services/scale-set/src/github-http.test.ts @@ -0,0 +1,58 @@ +const undiciMocks = vi.hoisted(() => ({ + close: vi.fn().mockResolvedValue(undefined), + createAgent: vi.fn(), +})); + +vi.mock('undici', () => ({ + Agent: class MockAgent { + constructor(options: unknown) { + undiciMocks.createAgent(options); + } + + close = undiciMocks.close; + }, +})); + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +import { createScaleSetGitHubHttp } from './github-http'; + +describe('scale-set GitHub HTTP isolation', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('uses the supplied verified fetch without changing process TLS settings', async () => { + const original = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + const fetchImplementation = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + const http = createScaleSetGitHubHttp(fetchImplementation); + + await http.fetch(true)('https://github.example/_apis/runtime/runnerscalesets'); + await http.close(); + + expect(fetchImplementation).toHaveBeenCalledWith('https://github.example/_apis/runtime/runnerscalesets'); + expect(undiciMocks.createAgent).not.toHaveBeenCalled(); + expect(process.env.NODE_TLS_REJECT_UNAUTHORIZED).toBe(original); + }); + + it('uses one scoped insecure dispatcher and closes it without mutating global TLS state', async () => { + const original = process.env.NODE_TLS_REJECT_UNAUTHORIZED; + const fetchImplementation = vi.fn().mockResolvedValue(new Response(null, { status: 204 })); + const http = createScaleSetGitHubHttp(fetchImplementation); + const first = http.fetch(false); + const second = http.fetch(false); + + await first('https://github.example/_apis/runtime/runnerscalesets', { method: 'GET' }); + await http.close(); + + expect(first).toBe(second); + expect(undiciMocks.createAgent).toHaveBeenCalledOnce(); + expect(undiciMocks.createAgent).toHaveBeenCalledWith({ connect: { rejectUnauthorized: false } }); + expect(fetchImplementation).toHaveBeenCalledWith( + 'https://github.example/_apis/runtime/runnerscalesets', + expect.objectContaining({ method: 'GET', dispatcher: expect.anything() }), + ); + expect(undiciMocks.close).toHaveBeenCalledOnce(); + expect(process.env.NODE_TLS_REJECT_UNAUTHORIZED).toBe(original); + }); +}); diff --git a/lambdas/services/scale-set/src/github-http.ts b/lambdas/services/scale-set/src/github-http.ts new file mode 100644 index 0000000000..85a60626b1 --- /dev/null +++ b/lambdas/services/scale-set/src/github-http.ts @@ -0,0 +1,34 @@ +import { Agent, type Dispatcher } from 'undici'; + +import type { ScaleSetFetch } from '@aws-github-runner/github-actions-scale-set'; + +type DispatcherRequestInit = RequestInit & { dispatcher: Dispatcher }; + +export interface ScaleSetGitHubHttp { + fetch(sslVerify: boolean): ScaleSetFetch; + close(): Promise; +} + +/** + * Creates fetch implementations whose TLS policy is scoped to one controller + * process. Disabling verification never mutates NODE_TLS_REJECT_UNAUTHORIZED + * or the global Undici dispatcher, so verified and unverified GHES runner + * configurations may safely share one grouped task. + */ +export function createScaleSetGitHubHttp(fetchImplementation: ScaleSetFetch = globalThis.fetch): ScaleSetGitHubHttp { + let insecureAgent: Agent | undefined; + let insecureFetch: ScaleSetFetch | undefined; + + return { + fetch(sslVerify) { + if (sslVerify) return fetchImplementation; + insecureAgent ??= new Agent({ connect: { rejectUnauthorized: false } }); + insecureFetch ??= async (input, init = {}) => + await fetchImplementation(input, { ...init, dispatcher: insecureAgent } as DispatcherRequestInit); + return insecureFetch; + }, + async close() { + await insecureAgent?.close(); + }, + }; +} diff --git a/lambdas/services/scale-set/src/health-server.test.ts b/lambdas/services/scale-set/src/health-server.test.ts new file mode 100644 index 0000000000..16e65ab953 --- /dev/null +++ b/lambdas/services/scale-set/src/health-server.test.ts @@ -0,0 +1,15 @@ +import { startScaleSetHealthServer } from './health-server'; + +describe('health server', () => { + it('separates liveness and readiness on loopback', async () => { + const health = { snapshot: vi.fn(() => ({ live: true, ready: false, state: 'degraded' })) }; + const server = await startScaleSetHealthServer(health, 0); + try { + await expect(fetch(`http://127.0.0.1:${server.port}/healthz`)).resolves.toMatchObject({ status: 200 }); + await expect(fetch(`http://127.0.0.1:${server.port}/readyz`)).resolves.toMatchObject({ status: 503 }); + await expect(fetch(`http://127.0.0.1:${server.port}/other`)).resolves.toMatchObject({ status: 404 }); + } finally { + await server.close(); + } + }); +}); diff --git a/lambdas/services/scale-set/src/health-server.ts b/lambdas/services/scale-set/src/health-server.ts new file mode 100644 index 0000000000..f9b967aaec --- /dev/null +++ b/lambdas/services/scale-set/src/health-server.ts @@ -0,0 +1,53 @@ +import { createServer, type Server } from 'node:http'; + +import type { ScaleSetControllerHealth } from './health'; + +export interface ScaleSetHealthServer { + port: number; + close(): Promise; +} + +export async function startScaleSetHealthServer( + health: Pick, + port: number, +): Promise { + const server = createServer((request, response) => { + response.setHeader('Cache-Control', 'no-store'); + response.setHeader('Connection', 'close'); + response.setHeader('Content-Type', 'application/json; charset=utf-8'); + response.setHeader('X-Content-Type-Options', 'nosniff'); + + if (request.method !== 'GET' || (request.url !== '/healthz' && request.url !== '/readyz')) { + response.statusCode = 404; + response.end(JSON.stringify({ status: 'not-found' })); + return; + } + + const snapshot = health.snapshot(); + const healthy = request.url === '/readyz' ? snapshot.ready : snapshot.live; + response.statusCode = healthy ? 200 : 503; + response.end(JSON.stringify(snapshot)); + }); + await listen(server, port); + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('health server did not bind a TCP address'); + return { + port: address.port, + close: async () => { + server.closeAllConnections(); + if (!server.listening) return; + await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); + }, + }; +} + +async function listen(server: Server, port: number): Promise { + await new Promise((resolve, reject) => { + const onError = (error: Error) => reject(error); + server.once('error', onError); + server.listen(port, '127.0.0.1', () => { + server.removeListener('error', onError); + resolve(); + }); + }); +} diff --git a/lambdas/services/scale-set/src/health.test.ts b/lambdas/services/scale-set/src/health.test.ts new file mode 100644 index 0000000000..b2d5294e11 --- /dev/null +++ b/lambdas/services/scale-set/src/health.test.ts @@ -0,0 +1,46 @@ +import { ScaleSetControllerHealth } from './health'; + +describe('ScaleSetControllerHealth', () => { + it('aggregates independent readiness while reconnect heartbeats stay live', () => { + let now = 0; + const health = new ScaleSetControllerHealth('group', ['a', 'b'], 100, () => now); + const a = health.reporter('a'); + const b = health.reporter('b'); + a.markSessionReady(); + b.markSessionReady(); + a.markProgress(); + b.markProgress(); + expect(health.snapshot()).toMatchObject({ state: 'ready', live: true, ready: true }); + + now = 200; + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'ready', live: true, ready: false }); + + a.markReconnecting(new Error('outage')); + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'reconnecting', live: true, ready: false }); + }); + + it('contains one terminal reconciler failure while another stays ready', () => { + const health = new ScaleSetControllerHealth('group', ['a', 'b'], 100); + health.reporter('a').markFailed(new TypeError('bad config')); + health.reporter('b').markProgress(); + expect(health.snapshot()).toMatchObject({ state: 'degraded', live: true, ready: false }); + expect(health.snapshot().reconcilers.a).toMatchObject({ state: 'failed', lastErrorName: 'TypeError' }); + }); + + it('marks all reporters stopping without reviving failures', () => { + const health = new ScaleSetControllerHealth('group', ['a'], 100); + health.reporter('a').markFailed(); + health.markStopping(); + expect(health.snapshot()).toMatchObject({ state: 'stopping', live: true, ready: false }); + expect(health.snapshot().reconcilers.a.state).toBe('failed'); + }); + + it('rejects duplicate and unknown reporter names', () => { + expect(() => new ScaleSetControllerHealth('g', [], 1)).toThrow('at least one'); + expect(() => new ScaleSetControllerHealth('g', ['a', 'a'], 1)).toThrow('duplicate'); + const health = new ScaleSetControllerHealth('g', ['a'], 1); + expect(() => health.reporter('b')).toThrow('unknown runner config'); + }); +}); diff --git a/lambdas/services/scale-set/src/health.ts b/lambdas/services/scale-set/src/health.ts new file mode 100644 index 0000000000..6cad43d10d --- /dev/null +++ b/lambdas/services/scale-set/src/health.ts @@ -0,0 +1,137 @@ +export type ScaleSetReconcilerState = 'starting' | 'ready' | 'reconnecting' | 'failed' | 'stopping'; +export type ScaleSetControllerState = 'starting' | 'ready' | 'degraded' | 'failed' | 'stopping'; + +export interface ScaleSetReconcilerHealthSnapshot { + state: ScaleSetReconcilerState; + live: boolean; + ready: boolean; + lastActivityAt: string; + consecutiveFailures: number; + lastErrorName?: string; +} + +export interface ScaleSetControllerHealthSnapshot { + groupName: string; + state: ScaleSetControllerState; + live: boolean; + ready: boolean; + reconcilers: Readonly>; +} + +export interface ScaleSetReconcilerStatusReporter { + markSessionReady(): void; + markProgress(): void; + markReconnecting(error?: unknown): void; + markFailed(error?: unknown): void; + markStopping(): void; +} + +interface MutableHealth { + state: ScaleSetReconcilerState; + lastActivityAt: number; + consecutiveFailures: number; + lastErrorName?: string; +} + +function errorName(error: unknown): string | undefined { + if (error === undefined) return undefined; + return error instanceof Error ? error.name : typeof error; +} + +export class ScaleSetControllerHealth { + private readonly states = new Map(); + private stopping = false; + + constructor( + readonly groupName: string, + runnerConfigNames: readonly string[], + private readonly staleAfterMs: number, + private readonly now: () => number = Date.now, + ) { + const startedAt = now(); + for (const name of runnerConfigNames) { + if (this.states.has(name)) throw new Error(`duplicate health reporter for ${JSON.stringify(name)}`); + this.states.set(name, { state: 'starting', lastActivityAt: startedAt, consecutiveFailures: 0 }); + } + if (this.states.size === 0) throw new Error('at least one reconciler health reporter is required'); + } + + reporter(runnerConfigName: string): ScaleSetReconcilerStatusReporter { + const state = this.states.get(runnerConfigName); + if (!state) throw new Error(`unknown runner config ${JSON.stringify(runnerConfigName)}`); + return { + markSessionReady: () => { + if (state.state === 'starting') { + state.state = 'ready'; + state.lastActivityAt = this.now(); + } else if (state.state === 'reconnecting') { + state.state = 'ready'; + } + }, + markProgress: () => { + if (state.state === 'failed' || state.state === 'stopping') return; + state.state = 'ready'; + state.lastActivityAt = this.now(); + state.consecutiveFailures = 0; + state.lastErrorName = undefined; + }, + markReconnecting: (error) => { + if (state.state === 'failed' || state.state === 'stopping') return; + state.state = 'reconnecting'; + state.lastActivityAt = this.now(); + state.consecutiveFailures += 1; + state.lastErrorName = errorName(error); + }, + markFailed: (error) => { + if (state.state === 'stopping') return; + state.state = 'failed'; + state.consecutiveFailures += 1; + state.lastErrorName = errorName(error); + }, + markStopping: () => { + if (state.state !== 'failed') state.state = 'stopping'; + }, + }; + } + + markStopping(): void { + this.stopping = true; + for (const name of this.states.keys()) this.reporter(name).markStopping(); + } + + snapshot(): ScaleSetControllerHealthSnapshot { + const now = this.now(); + const reconcilers: Record = Object.create(null) as Record< + string, + ScaleSetReconcilerHealthSnapshot + >; + for (const [name, state] of this.states) { + const stale = now - state.lastActivityAt > this.staleAfterMs; + // Staleness means the reconciler is not ready, but it is not a process + // liveness failure. A single bounded GitHub/AWS request can legitimately + // outlive the readiness window; restarting the task would only churn its + // message sessions and reset the provider retry policy. + const live = state.state === 'stopping' || state.state !== 'failed'; + reconcilers[name] = { + state: state.state, + live, + ready: state.state === 'ready' && !stale, + lastActivityAt: new Date(state.lastActivityAt).toISOString(), + consecutiveFailures: state.consecutiveFailures, + ...(state.lastErrorName === undefined ? {} : { lastErrorName: state.lastErrorName }), + }; + } + const values = Object.values(reconcilers); + const liveCount = values.filter(({ live }) => live).length; + const readyCount = values.filter(({ ready }) => ready).length; + const live = this.stopping || liveCount > 0; + const ready = !this.stopping && readyCount === values.length; + let state: ScaleSetControllerState; + if (this.stopping) state = 'stopping'; + else if (ready) state = 'ready'; + else if (liveCount === 0) state = 'failed'; + else if (values.every((value) => value.state === 'starting')) state = 'starting'; + else state = 'degraded'; + return { groupName: this.groupName, state, live, ready, reconcilers }; + } +} diff --git a/lambdas/services/scale-set/src/index.ts b/lambdas/services/scale-set/src/index.ts new file mode 100644 index 0000000000..6074b49499 --- /dev/null +++ b/lambdas/services/scale-set/src/index.ts @@ -0,0 +1,9 @@ +export * from './config'; +export * from './controller'; +export * from './credentials'; +export * from './health'; +export * from './health-server'; +export * from './lifecycle'; +export * from './logger'; +export * from './parameter-store'; +export * from './reconciler'; diff --git a/lambdas/services/scale-set/src/lifecycle.test.ts b/lambdas/services/scale-set/src/lifecycle.test.ts new file mode 100644 index 0000000000..a43c31fe5c --- /dev/null +++ b/lambdas/services/scale-set/src/lifecycle.test.ts @@ -0,0 +1,45 @@ +import { ScaleSetServiceRuntime } from './lifecycle'; + +describe('ScaleSetServiceRuntime', () => { + it('starts once and performs idempotent bounded shutdown', async () => { + const health = { markStopping: vi.fn(), snapshot: vi.fn() }; + const run = vi.fn(async (signal: AbortSignal) => { + if (!signal.aborted) + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); + }); + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 100 }, { run, health } as never); + const completion = runtime.run(); + expect(() => runtime.run()).toThrow('already started'); + const shutdown = runtime.shutdown(); + expect(runtime.shutdown()).toBe(shutdown); + await shutdown; + await completion; + expect(health.markStopping).toHaveBeenCalledOnce(); + }); + + it('allows shutdown before start and prevents a later start', async () => { + const health = { markStopping: vi.fn() }; + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 100 }, { run: vi.fn(), health } as never); + await runtime.shutdown(); + expect(() => runtime.run()).toThrow('already stopping'); + }); + + it('rejects when a controller ignores cancellation past the timeout', async () => { + vi.useFakeTimers(); + try { + const health = { markStopping: vi.fn() }; + const runtime = new ScaleSetServiceRuntime({ shutdownTimeoutMs: 10 }, { + run: vi.fn(() => new Promise(() => undefined)), + health, + } as never); + void runtime.run(); + await Promise.resolve(); + const shutdown = runtime.shutdown(); + const expectation = expect(shutdown).rejects.toThrow('did not stop within 10ms'); + await vi.advanceTimersByTimeAsync(10); + await expectation; + } finally { + vi.useRealTimers(); + } + }); +}); diff --git a/lambdas/services/scale-set/src/lifecycle.ts b/lambdas/services/scale-set/src/lifecycle.ts new file mode 100644 index 0000000000..d4c4a600c0 --- /dev/null +++ b/lambdas/services/scale-set/src/lifecycle.ts @@ -0,0 +1,51 @@ +import type { ScaleSetServiceConfig } from './config'; +import type { ScaleSetController } from './controller'; + +export class ScaleSetServiceRuntime { + private readonly abortController = new AbortController(); + private completion: Promise | undefined; + private shutdownCompletion: Promise | undefined; + + constructor( + private readonly config: Pick, + private readonly controller: Pick, + ) {} + + get health() { + return this.controller.health; + } + + run(): Promise { + if (this.shutdownCompletion !== undefined) throw new Error('Scale-set service runtime is already stopping'); + if (this.completion !== undefined) throw new Error('Scale-set service runtime has already started'); + this.completion = Promise.resolve().then(async () => { + if (!this.abortController.signal.aborted) await this.controller.run(this.abortController.signal); + }); + return this.completion; + } + + shutdown(reason: unknown = new Error('Scale-set service shutdown requested')): Promise { + this.shutdownCompletion ??= this.shutdownOnce(reason); + return this.shutdownCompletion; + } + + private async shutdownOnce(reason: unknown): Promise { + this.controller.health.markStopping(); + this.abortController.abort(reason); + if (this.completion === undefined) return; + let timeout: ReturnType | undefined; + try { + await Promise.race([ + this.completion, + new Promise((_resolve, reject) => { + timeout = setTimeout( + () => reject(new Error(`Scale-set controller did not stop within ${this.config.shutdownTimeoutMs}ms`)), + this.config.shutdownTimeoutMs, + ); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } + } +} diff --git a/lambdas/services/scale-set/src/logger.test.ts b/lambdas/services/scale-set/src/logger.test.ts new file mode 100644 index 0000000000..0a98b76ec8 --- /dev/null +++ b/lambdas/services/scale-set/src/logger.test.ts @@ -0,0 +1,26 @@ +import { logger, sanitizeLogAttributes } from './logger'; + +describe('redacted structured logging', () => { + it('redacts nested secrets and strips log-injection characters', () => { + expect( + sanitizeLogAttributes({ + runnerConfig: 'linux\nforged', + privateKey: 'secret', + nested: { authorization: 'Bearer secret', safe: 'ok' }, + }), + ).toEqual({ + runnerConfig: 'linux forged', + privateKey: '[REDACTED]', + nested: { authorization: '[REDACTED]', safe: 'ok' }, + }); + }); + + it('logs errors without their potentially sensitive message', () => { + const spy = vi.spyOn(console, 'error').mockImplementation(() => undefined); + logger.error('failed', { error: new Error('token=secret') }); + expect(spy).toHaveBeenCalledOnce(); + expect(spy.mock.calls[0][0]).not.toContain('token=secret'); + expect(JSON.parse(spy.mock.calls[0][0] as string)).toMatchObject({ level: 'error', event: 'failed' }); + spy.mockRestore(); + }); +}); diff --git a/lambdas/services/scale-set/src/logger.ts b/lambdas/services/scale-set/src/logger.ts new file mode 100644 index 0000000000..daf0597c68 --- /dev/null +++ b/lambdas/services/scale-set/src/logger.ts @@ -0,0 +1,57 @@ +const REDACTED = '[REDACTED]'; +const SENSITIVE_KEY = /(authorization|credential|encodedjit|jitconfig|password|private.?key|secret|sessionid|token)/i; +const MAX_LOG_STRING_LENGTH = 1024; +const MAX_LOG_DEPTH = 4; + +export interface ScaleSetLogger { + info(event: string, attributes?: Readonly>): void; + warn(event: string, attributes?: Readonly>): void; + error(event: string, attributes?: Readonly>): void; +} + +function sanitizeString(value: string): string { + return value.replace(/[\r\n\u2028\u2029]/g, ' ').slice(0, MAX_LOG_STRING_LENGTH); +} + +function sanitize(value: unknown, key: string, depth: number): unknown { + if (SENSITIVE_KEY.test(key)) return REDACTED; + if (depth > MAX_LOG_DEPTH) return '[TRUNCATED]'; + if (value === null || typeof value === 'boolean' || typeof value === 'number') return value; + if (typeof value === 'string') return sanitizeString(value); + if (value instanceof Error) { + const status = 'status' in value && typeof value.status === 'number' ? value.status : undefined; + const code = 'code' in value && typeof value.code === 'string' ? sanitizeString(value.code) : undefined; + return { name: sanitizeString(value.name), ...(status === undefined ? {} : { status }), ...(code ? { code } : {}) }; + } + if (Array.isArray(value)) return value.slice(0, 100).map((item) => sanitize(item, key, depth + 1)); + if (typeof value === 'object') { + const result: Record = Object.create(null) as Record; + for (const [childKey, childValue] of Object.entries(value).slice(0, 100)) { + result[sanitizeString(childKey)] = sanitize(childValue, childKey, depth + 1); + } + return result; + } + return sanitizeString(typeof value); +} + +export function sanitizeLogAttributes(attributes: Readonly> = {}): Record { + return sanitize(attributes, '', 0) as Record; +} + +function write(level: 'info' | 'warn' | 'error', event: string, attributes?: Readonly>): void { + const record = JSON.stringify({ + timestamp: new Date().toISOString(), + level, + event: sanitizeString(event), + ...sanitizeLogAttributes(attributes), + }); + if (level === 'error') console.error(record); + else if (level === 'warn') console.warn(record); + else console.info(record); +} + +export const logger: ScaleSetLogger = { + info: (event, attributes) => write('info', event, attributes), + warn: (event, attributes) => write('warn', event, attributes), + error: (event, attributes) => write('error', event, attributes), +}; diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts new file mode 100644 index 0000000000..fb5425a4b7 --- /dev/null +++ b/lambdas/services/scale-set/src/main.ts @@ -0,0 +1,85 @@ +import { GitHubActionsScaleSetClient } from '@aws-github-runner/github-actions-scale-set'; +import { createScaleSetComputeProviderRegistry } from '@aws-github-runner/compute-providers/scale-set'; + +import { parseScaleSetServiceConfig } from './config'; +import { ScaleSetController } from './controller'; +import { createGitHubAppAccessTokenProvider } from './credentials'; +import { createScaleSetGitHubHttp } from './github-http'; +import { startScaleSetHealthServer, type ScaleSetHealthServer } from './health-server'; +import { ScaleSetServiceRuntime } from './lifecycle'; +import { logger } from './logger'; +import { createDefaultControllerManifestLoader, defaultParameterStore } from './parameter-store'; +import { abortableSleep, TtlScaleSetRunnerInventoryCache, type ScaleSetReconcilerDependencies } from './reconciler'; + +async function main(): Promise { + const serviceConfig = parseScaleSetServiceConfig(process.env); + const manifest = await createDefaultControllerManifestLoader().load(serviceConfig); + const computeProviders = createScaleSetComputeProviderRegistry(); + const githubHttp = createScaleSetGitHubHttp(); + const dependencies: ScaleSetReconcilerDependencies = { + computeProviders, + createAccessTokenProvider: async (config) => + await createGitHubAppAccessTokenProvider( + config.githubApp, + config.githubConfigUrl, + config.forceGhes, + defaultParameterStore, + githubHttp.fetch(config.sslVerify), + ), + createClient: (config, accessTokenProvider) => + new GitHubActionsScaleSetClient({ + gitHubConfigUrl: config.githubConfigUrl, + accessTokenProvider, + fetch: githubHttp.fetch(config.sslVerify), + forceGhes: config.forceGhes, + systemInfo: { + system: config.userAgent ?? 'github-aws-runners', + version: '1', + scaleSetId: config.scaleSetId, + subsystem: 'scale-set-controller', + }, + }), + logger, + sleep: abortableSleep, + random: Math.random, + closeSignal: AbortSignal.timeout, + runnerInventory: new TtlScaleSetRunnerInventoryCache(), + }; + const controller = new ScaleSetController(manifest, serviceConfig, dependencies, logger); + const runtime = new ScaleSetServiceRuntime(serviceConfig, controller); + let healthServer: ScaleSetHealthServer | undefined; + + const shutdown = (signal: NodeJS.Signals) => { + logger.info('scale_set_controller_shutdown_requested', { signal, groupName: manifest.groupName }); + void runtime.shutdown(new Error(`received ${signal}`)).catch((error) => { + logger.error('scale_set_controller_shutdown_failed', { error, groupName: manifest.groupName }); + process.exitCode = 1; + }); + }; + const onSigterm = () => shutdown('SIGTERM'); + const onSigint = () => shutdown('SIGINT'); + process.once('SIGTERM', onSigterm); + process.once('SIGINT', onSigint); + + try { + healthServer = await startScaleSetHealthServer(runtime.health, serviceConfig.healthPort); + logger.info('scale_set_controller_started', { + groupName: manifest.groupName, + revision: manifest.revision, + reconcilerCount: manifest.reconcilers.length, + healthPort: healthServer.port, + }); + await runtime.run(); + } finally { + await runtime.shutdown().catch(() => undefined); + await healthServer?.close(); + await githubHttp.close(); + process.removeListener('SIGTERM', onSigterm); + process.removeListener('SIGINT', onSigint); + } +} + +void main().catch((error) => { + logger.error('scale_set_controller_fatal_failure', { error }); + process.exitCode = 1; +}); diff --git a/lambdas/services/scale-set/src/parameter-store.test.ts b/lambdas/services/scale-set/src/parameter-store.test.ts new file mode 100644 index 0000000000..f18cc2ef7e --- /dev/null +++ b/lambdas/services/scale-set/src/parameter-store.test.ts @@ -0,0 +1,78 @@ +import { createControllerManifestLoader, type ParametersByPathClient } from './parameter-store'; +import type { ScaleSetServiceConfig } from './config'; + +function leaf(name: string, id: number): string { + return JSON.stringify({ + schemaVersion: 1, + runnerConfigName: name, + githubConfigUrl: 'https://github.com/example', + scaleSetId: id, + expectedScaleSetName: name, + expectedRunnerGroupId: null, + minRunners: 0, + maxRunners: 10, + sslVerify: true, + githubApp: { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', + }, + computeProvider: { type: 'ec2', configuration: {} }, + }); +} + +const config: ScaleSetServiceConfig = { + groupName: 'group', + groupConfigPath: '/groups/group', + groupRevision: 'rev-1', + healthPort: 8080, + healthStaleAfterMs: 1000, + shutdownTimeoutMs: 1000, + sessionCloseTimeoutMs: 1000, + reconnectInitialBackoffMs: 100, + reconnectMaxBackoffMs: 1000, +}; + +describe('createControllerManifestLoader', () => { + it('paginates direct children, sorts them, and returns a versioned group', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/b', Value: leaf('b', 2) }], NextToken: 'next' }) + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/a', Value: leaf('a', 1) }] }); + const manifest = await createControllerManifestLoader({ send } as ParametersByPathClient).load(config); + expect(manifest).toMatchObject({ version: 1, groupName: 'group', revision: 'rev-1' }); + expect(manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName)).toEqual(['a', 'b']); + expect(send).toHaveBeenCalledTimes(2); + }); + + it('uses the injected inline manifest loader path for local tests', async () => { + const inline = JSON.stringify({ version: 1, groupName: 'local', reconcilers: [JSON.parse(leaf('a', 1))] }); + const send = vi.fn(); + await expect( + createControllerManifestLoader({ send } as ParametersByPathClient).load({ ...config, manifest: inline }), + ).resolves.toMatchObject({ + groupName: 'local', + }); + expect(send).not.toHaveBeenCalled(); + }); + + it.each([ + [{ Parameters: [] }, 'contains no runner configs'], + [{ Parameters: [{ Name: '/groups/group/nested/a', Value: leaf('a', 1) }] }, 'outside the direct group path'], + [{ Parameters: [{ Name: '/groups/group/wrong', Value: leaf('a', 1) }] }, 'must match runnerConfigName'], + [{ Parameters: [{ Name: '/groups/group/a', Value: '{' }] }, 'contains invalid JSON'], + [{ Parameters: [{ Name: undefined, Value: leaf('a', 1) }] }, 'incomplete parameter'], + ])('rejects malformed SSM group pages', async (page, message) => { + await expect( + createControllerManifestLoader({ send: vi.fn().mockResolvedValue(page) }).load(config), + ).rejects.toThrow(message); + }); + + it('rejects repeated pagination tokens', async () => { + const send = vi + .fn() + .mockResolvedValueOnce({ Parameters: [{ Name: '/groups/group/a', Value: leaf('a', 1) }], NextToken: 'same' }) + .mockResolvedValueOnce({ Parameters: [], NextToken: 'same' }); + await expect(createControllerManifestLoader({ send }).load(config)).rejects.toThrow('repeated token'); + }); +}); diff --git a/lambdas/services/scale-set/src/parameter-store.ts b/lambdas/services/scale-set/src/parameter-store.ts new file mode 100644 index 0000000000..ce07141bd7 --- /dev/null +++ b/lambdas/services/scale-set/src/parameter-store.ts @@ -0,0 +1,131 @@ +import { GetParametersByPathCommand, SSMClient } from '@aws-sdk/client-ssm'; + +import { getParameters } from '@aws-github-runner/aws-ssm-util'; + +import { + MAX_MANIFEST_BYTES, + SCALE_SET_CONTROLLER_MANIFEST_VERSION, + ScaleSetConfigurationError, + parseScaleSetControllerManifest, + parseScaleSetReconcilerConfig, + validateUniqueReconcilers, + type ScaleSetControllerManifest, + type ScaleSetServiceConfig, +} from './config'; +import type { ParameterStore } from './credentials'; + +const MAX_GROUP_PARAMETERS = 1000; +const MAX_PARAMETER_BYTES = 64 * 1024; +const MAX_GROUP_BYTES = 4 * 1024 * 1024; + +export const defaultParameterStore: ParameterStore = { + get: async (names) => await getParameters([...names]), +}; + +export interface ControllerManifestLoader { + load(config: ScaleSetServiceConfig): Promise; +} + +export interface ParametersByPathClient { + send(command: GetParametersByPathCommand): Promise<{ + Parameters?: Array<{ Name?: string; Value?: string }>; + NextToken?: string; + }>; +} + +export function createControllerManifestLoader(client: ParametersByPathClient): ControllerManifestLoader { + return { + load: async (config) => { + if (config.manifest !== undefined) return parseScaleSetControllerManifest(config.manifest); + if (!config.groupConfigPath || !config.groupName || !config.groupRevision) { + throw new ScaleSetConfigurationError('SSM group configuration source is incomplete'); + } + const prefix = `${config.groupConfigPath.replace(/\/$/, '')}/`; + const parameters: Array<{ name: string; value: string }> = []; + const seenTokens = new Set(); + let nextToken: string | undefined; + let totalBytes = 0; + do { + if (nextToken !== undefined && seenTokens.has(nextToken)) { + throw new ScaleSetConfigurationError('SSM pagination returned a repeated token'); + } + if (nextToken !== undefined) seenTokens.add(nextToken); + const response = await client.send( + new GetParametersByPathCommand({ + Path: config.groupConfigPath, + Recursive: false, + WithDecryption: false, + MaxResults: 10, + ...(nextToken === undefined ? {} : { NextToken: nextToken }), + }), + ); + for (const parameter of response.Parameters ?? []) { + if (!parameter.Name || parameter.Value === undefined) { + throw new ScaleSetConfigurationError('SSM group configuration returned an incomplete parameter'); + } + if (!parameter.Name.startsWith(prefix) || parameter.Name.slice(prefix.length).includes('/')) { + throw new ScaleSetConfigurationError( + 'SSM group configuration returned a parameter outside the direct group path', + ); + } + const size = Buffer.byteLength(parameter.Value, 'utf8'); + if (size > MAX_PARAMETER_BYTES || size > MAX_MANIFEST_BYTES) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(parameter.Name)} is too large`, + ); + } + totalBytes += size; + if (totalBytes > MAX_GROUP_BYTES) + throw new ScaleSetConfigurationError('SSM controller group configuration is too large'); + parameters.push({ name: parameter.Name, value: parameter.Value }); + if (parameters.length > MAX_GROUP_PARAMETERS) { + throw new ScaleSetConfigurationError(`SSM controller group exceeds ${MAX_GROUP_PARAMETERS} runner configs`); + } + } + nextToken = response.NextToken; + } while (nextToken !== undefined); + + if (parameters.length === 0) + throw new ScaleSetConfigurationError('SSM controller group contains no runner configs'); + parameters.sort((left, right) => left.name.localeCompare(right.name)); + const reconcilers = parameters.map(({ name, value }, index) => { + let parsed: unknown; + try { + parsed = JSON.parse(value) as unknown; + } catch (error) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(name)} contains invalid JSON`, + { + cause: error, + }, + ); + } + const reconciler = parseScaleSetReconcilerConfig(parsed, index, config.groupName as string, 'ssmRunnerConfigs'); + const leafName = name.slice(prefix.length); + if (leafName !== reconciler.runnerConfigName) { + throw new ScaleSetConfigurationError( + `runner config parameter ${JSON.stringify(name)} must match runnerConfigName ${JSON.stringify(reconciler.runnerConfigName)}`, + ); + } + return reconciler; + }); + validateUniqueReconcilers(reconcilers); + return { + version: SCALE_SET_CONTROLLER_MANIFEST_VERSION, + groupName: config.groupName, + revision: config.groupRevision, + reconcilers, + }; + }, + }; +} + +export function createDefaultControllerManifestLoader(): ControllerManifestLoader { + return createControllerManifestLoader( + new SSMClient({ + region: process.env.AWS_REGION, + maxAttempts: 10, + retryMode: 'adaptive', + }), + ); +} diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts new file mode 100644 index 0000000000..a83baa87c9 --- /dev/null +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -0,0 +1,355 @@ +import type { MessageSessionClient, RunnerScaleSetMessage } from '@aws-github-runner/github-actions-scale-set'; +import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '@aws-github-runner/compute-providers/scale-set'; + +import type { ScaleSetReconcilerConfig, ScaleSetServiceConfig } from './config'; +import type { ScaleSetReconcilerStatusReporter } from './health'; +import { + ScaleSetReconciler, + TtlScaleSetRunnerInventoryCache, + calculateDesiredRunners, + validateProviderResult, + type ScaleSetReconcilerClient, + type ScaleSetReconcilerDependencies, +} from './reconciler'; + +const config: ScaleSetReconcilerConfig = { + schemaVersion: 1, + runnerConfigName: 'linux', + scaleSetId: 42, + expectedScaleSetName: 'linux', + githubConfigUrl: 'https://github.com/example', + githubApp: { + appIdParameterName: '/app/id', + privateKeyParameterName: '/app/key', + installationIdParameterName: '/app/installation', + }, + computeProvider: { type: 'ec2', configuration: {} }, + minRunners: 0, + maxRunners: 10, + bootTimeoutMinutes: 10, + sessionOwner: 'group.linux', + workFolder: '_work', + forceGhes: false, +}; + +const serviceConfig: Pick< + ScaleSetServiceConfig, + 'sessionCloseTimeoutMs' | 'reconnectInitialBackoffMs' | 'reconnectMaxBackoffMs' +> = { sessionCloseTimeoutMs: 100, reconnectInitialBackoffMs: 1, reconnectMaxBackoffMs: 10 }; + +function result(overrides: Partial = {}): ScaleSetReconcileResult { + return { + status: 'converged', + desiredRunners: 1, + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + ...overrides, + }; +} + +function reporter(): ScaleSetReconcilerStatusReporter { + return { + markSessionReady: vi.fn(), + markProgress: vi.fn(), + markReconnecting: vi.fn(), + markFailed: vi.fn(), + markStopping: vi.fn(), + }; +} + +function message(): RunnerScaleSetMessage { + return { + messageId: 7, + statistics: { + totalAvailableJobs: 1, + totalAcquiredJobs: 0, + totalAssignedJobs: 1, + totalRunningJobs: 0, + totalRegisteredRunners: 1, + totalBusyRunners: 0, + totalIdleRunners: 1, + }, + jobAvailableMessages: [{ runnerRequestId: 99 } as RunnerScaleSetMessage['jobAvailableMessages'][number]], + jobAssignedMessages: [], + jobStartedMessages: [ + { runnerId: 5, runnerName: 'runner-5' } as RunnerScaleSetMessage['jobStartedMessages'][number], + ], + jobCompletedMessages: [], + }; +} + +function fixture(options: { + session: Partial & { session: MessageSessionClient['session'] }; + reconcile?: ScaleSetComputeProvider['reconcile']; +}) { + const computeProvider: ScaleSetComputeProvider = { + reconcile: options.reconcile ?? vi.fn().mockResolvedValue(result()), + }; + const client: ScaleSetReconcilerClient = { + getRunnerScaleSetById: vi.fn().mockResolvedValue({ id: 42, name: 'linux' }), + createMessageSessionClient: vi.fn().mockResolvedValue(options.session as MessageSessionClient), + generateJitRunnerConfig: vi.fn(), + getGitHubRunner: vi.fn().mockResolvedValue({ id: 5, name: 'runner-5', status: 'online', busy: false }), + getRunnerByName: vi.fn(), + listGitHubRunners: vi.fn().mockResolvedValue([]), + listRunners: vi.fn().mockResolvedValue([]), + removeRunner: vi.fn(), + }; + const dependencies: ScaleSetReconcilerDependencies = { + createAccessTokenProvider: vi.fn().mockResolvedValue(async () => ({ token: 'not-a-real-token' })), + createClient: vi.fn().mockReturnValue(client), + computeProviders: { create: vi.fn().mockReturnValue(computeProvider) }, + logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + sleep: vi.fn(async (_delay, signal) => { + if (!signal.aborted) + await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); + }), + random: () => 0, + closeSignal: () => new AbortController().signal, + runnerInventory: new TtlScaleSetRunnerInventoryCache(), + }; + return { client, computeProvider, dependencies }; +} + +describe('ScaleSetReconciler', () => { + it('acknowledges only after acquisition, lifecycle observation, and successful reconciliation', async () => { + const order: string[] = []; + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(async () => { + order.push('acquire'); + return [99]; + }), + deleteMessage: vi.fn(async () => { + order.push('delete'); + abort.abort(); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async (request) => { + order.push('reconcile'); + expect(request.runnerInventoryComplete).toBe(false); + expect(request.bootTimeoutMinutes).toBe(10); + expect(request.runnerStates).toContainEqual( + expect.objectContaining({ runnerId: 5, runnerName: 'runner-5', lifecycle: 'started' }), + ); + return result(); + }); + const { dependencies } = fixture({ session, reconcile }); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + expect(order).toEqual(['acquire', 'reconcile', 'delete']); + expect(dependencies.computeProviders.create).toHaveBeenCalledWith('ec2', { + runnerConfigName: 'linux', + scaleSetId: 42, + githubScope: 'https://github.com/example', + configuration: {}, + }); + }); + + it('does not query public or Actions runner inventory on steady-state empty polls', async () => { + const abort = new AbortController(); + let calls = 0; + const reconcile = vi.fn(async () => { + calls += 1; + if (calls === 2) abort.abort(); + return result({ desiredRunners: 0, currentRunners: 0 }); + }); + const session = { + session: { statistics: { ...message().statistics, totalAssignedJobs: 0 } }, + getMessage: vi.fn().mockResolvedValue(null), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ session, reconcile }); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + expect(reconcile).toHaveBeenCalledTimes(2); + expect(client.listGitHubRunners).not.toHaveBeenCalled(); + expect(client.listRunners).not.toHaveBeenCalled(); + }); + + it('performs the typed inventory second pass whenever requested, including at desired physical capacity', async () => { + const abort = new AbortController(); + const reconcile = vi + .fn() + .mockResolvedValueOnce( + result({ + status: 'retained', + desiredRunners: 1, + currentRunners: 1, + needsRunnerInventory: true, + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, + errors: [], + }), + ) + .mockImplementationOnce(async (request) => { + expect(request.runnerInventoryComplete).toBe(true); + expect(request.runnerStates).toContainEqual({ + runnerId: 5, + runnerName: 'runner-5', + scaleSetId: 42, + status: 'online', + busy: false, + lifecycle: 'unknown', + }); + abort.abort(); + return result({ desiredRunners: 1, currentRunners: 1 }); + }); + const session = { + session: { statistics: message().statistics }, + close: vi.fn(), + }; + const { client, dependencies } = fixture({ session, reconcile }); + vi.mocked(client.listRunners).mockResolvedValue([{ id: 5, name: 'runner-5', runnerScaleSetId: 42 }]); + vi.mocked(client.listGitHubRunners).mockResolvedValue([{ id: 5, name: 'runner-5', status: 'online', busy: false }]); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + expect(reconcile.mock.calls[0]?.[0]).toEqual(expect.objectContaining({ runnerInventoryComplete: false })); + expect(client.listGitHubRunners).toHaveBeenCalledTimes(1); + expect(client.listRunners).toHaveBeenCalledTimes(1); + }); + + it('rejects a provider that requests another inventory after the complete second pass', async () => { + const abort = new AbortController(); + const reconcile = vi.fn().mockResolvedValue( + result({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, + }), + ); + const session = { session: { statistics: message().statistics }, close: vi.fn() }; + const { dependencies } = fixture({ session, reconcile }); + dependencies.sleep = vi.fn(async () => abort.abort()); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(reconcile).toHaveBeenCalledTimes(2); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ + message: 'scale-set compute provider requested inventory after a complete inventory pass', + }), + ); + }); + + it('leaves a message unacknowledged when reconciliation fails', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const { dependencies } = fixture({ session, reconcile: vi.fn().mockRejectedValue(new Error('temporary')) }); + dependencies.sleep = vi.fn(async () => abort.abort()); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + expect(session.deleteMessage).not.toHaveBeenCalled(); + }); + + it('re-fetches exact state in the serialized loop and acknowledges a typed busy retention', async () => { + const abort = new AbortController(); + const order: string[] = []; + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(async () => { + order.push('acquire'); + return [99]; + }), + deleteMessage: vi.fn(async () => { + order.push('delete'); + abort.abort(); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async (request) => { + order.push('reconcile'); + await expect(request.removeRunner({ runnerId: 5, runnerName: 'runner-5', scaleSetId: 42 })).resolves.toEqual({ + status: 'retained_busy', + }); + return result({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, + }); + }); + const { client, dependencies } = fixture({ session, reconcile }); + vi.mocked(client.getRunnerByName).mockImplementation(async () => { + order.push('actions-refetch'); + return { id: 5, name: 'runner-5', runnerScaleSetId: 42 }; + }); + vi.mocked(client.getGitHubRunner).mockImplementation(async () => { + order.push('github-refetch'); + return { id: 5, name: 'runner-5', status: 'online', busy: true }; + }); + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); + + expect(order).toEqual(['acquire', 'reconcile', 'actions-refetch', 'github-refetch', 'delete']); + expect(client.removeRunner).not.toHaveBeenCalled(); + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(session.acquireJobs).toHaveBeenCalledTimes(1); + }); + + it('bounds the lifecycle cache per reconciler', () => { + const { dependencies } = fixture({ session: { session: {}, close: vi.fn() } }); + const reconciler = new ScaleSetReconciler(config, serviceConfig, dependencies) as unknown as { + rememberLifecycle(id: number, name: string, lifecycle: 'started'): void; + lifecycle: Map; + }; + for (let index = 0; index < 1100; index += 1) reconciler.rememberLifecycle(index + 1, `runner-${index}`, 'started'); + expect(reconciler.lifecycle.size).toBe(1000); + expect(reconciler.lifecycle.has('runner-0')).toBe(false); + }); +}); + +describe('reconciler helpers', () => { + it('calculates bounded desired capacity', () => { + expect(calculateDesiredRunners(5, 2, 6)).toBe(6); + expect(calculateDesiredRunners(8, 2, 5)).toBe(8); + expect(() => calculateDesiredRunners(-1, 0, 1)).toThrow('non-negative integer'); + }); + + it('shares successful inventory loads and retries failed loads', async () => { + let now = 0; + const cache = new TtlScaleSetRunnerInventoryCache(100, () => now); + const loader = vi.fn().mockResolvedValue([{ id: 1, name: 'a', status: 'online', busy: false }]); + await Promise.all([cache.get('scope', loader), cache.get('scope', loader)]); + expect(loader).toHaveBeenCalledTimes(1); + now = 101; + await cache.get('scope', loader); + expect(loader).toHaveBeenCalledTimes(2); + await expect(cache.get('failed', vi.fn().mockRejectedValue(new Error('nope')))).rejects.toThrow('nope'); + }); + + it.each([ + { status: 'unexpected' }, + { needsRunnerInventory: 'yes' }, + { actions: { launched: 0, terminated: 0, retainedBusy: -1, retainedUnknown: 0 } }, + { errors: [{ operation: 'shell', code: 'BAD', retryable: false }] }, + { errors: [{ operation: 'list', code: 'contains spaces', retryable: false }] }, + { errors: [{ operation: 'list', code: 'BAD!CODE', retryable: false }] }, + { errors: [{ operation: 'list', code: 'BAD\nCODE', retryable: false }] }, + ])('rejects malformed compute-provider result metadata: %o', (overrides) => { + expect(() => validateProviderResult({ ...result(), ...overrides } as ScaleSetReconcileResult, 1)).toThrow( + /scale-set compute provider returned (?:an? )?invalid/, + ); + }); + + it('accepts bounded provider and AWS error codes', () => { + expect(() => + validateProviderResult( + result({ + errors: [ + { operation: 'list', code: 'AccessDeniedException', retryable: false }, + { operation: 'launch', code: 'ThrottlingException', retryable: true }, + ], + }), + 1, + ), + ).not.toThrow(); + }); +}); diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts new file mode 100644 index 0000000000..f34c931926 --- /dev/null +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -0,0 +1,599 @@ +import { + GitHubActionsScaleSetClient, + isScaleSetHttpError, + ScaleSetProtocolError, + type AccessToken, + type GitHubRunnerReference, + type MessageSessionClient, + type RunnerScaleSetMessage, + type RunnerScaleSetStatistic, + type ScaleSetRunnerState as GitHubScaleSetRunnerState, +} from '@aws-github-runner/github-actions-scale-set'; +import type { + ScaleSetComputeProvider, + ScaleSetComputeProviderFactoryInput, + ScaleSetReconcileResult, + ScaleSetRunnerLifecycle, + ScaleSetRunnerState, +} from '@aws-github-runner/compute-providers/scale-set'; + +import { ScaleSetConfigurationError, type ScaleSetReconcilerConfig, type ScaleSetServiceConfig } from './config'; +import type { ScaleSetReconcilerStatusReporter } from './health'; +import type { ScaleSetLogger } from './logger'; + +const MAX_JIT_CONFIGURATION_BYTES = 1024 * 1024; +const SCALE_SET_INVENTORY_TTL_MS = 60_000; + +export interface ScaleSetComputeProviderFactory { + create(type: string, input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider; +} + +export type ScaleSetReconcilerClient = Pick< + GitHubActionsScaleSetClient, + | 'createMessageSessionClient' + | 'generateJitRunnerConfig' + | 'getGitHubRunner' + | 'getRunnerScaleSetById' + | 'getRunnerByName' + | 'listGitHubRunners' + | 'listRunners' + | 'removeRunner' +>; + +export interface ScaleSetReconcilerDependencies { + createAccessTokenProvider(config: ScaleSetReconcilerConfig): Promise<() => Promise>; + createClient(config: ScaleSetReconcilerConfig, provider: () => Promise): ScaleSetReconcilerClient; + computeProviders: ScaleSetComputeProviderFactory; + logger: ScaleSetLogger; + sleep(delayMs: number, signal: AbortSignal): Promise; + random(): number; + closeSignal(timeoutMs: number): AbortSignal; + runnerInventory: ScaleSetRunnerInventoryCache; +} + +export interface ScaleSetRunnerInventoryCache { + get(key: string, loader: () => Promise): Promise; +} + +export class TtlScaleSetRunnerInventoryCache implements ScaleSetRunnerInventoryCache { + private readonly entries = new Map }>(); + + constructor( + private readonly ttlMs = 60_000, + private readonly now: () => number = Date.now, + ) {} + + async get( + key: string, + loader: () => Promise, + ): Promise { + const current = this.entries.get(key); + if (current !== undefined && current.expiresAt > this.now()) return await current.value; + const value = loader(); + this.entries.set(key, { expiresAt: this.now() + this.ttlMs, value }); + try { + return await value; + } catch (error) { + if (this.entries.get(key)?.value === value) this.entries.delete(key); + throw error; + } + } +} + +interface LifecycleObservation { + runnerId: number; + runnerName: string; + scaleSetId: number; + lifecycle: ScaleSetRunnerLifecycle; +} + +export class ScaleSetProviderReconciliationError extends Error { + constructor( + readonly result: ScaleSetReconcileResult, + readonly retryable: boolean, + ) { + super(`scale-set compute provider returned ${result.status}`); + this.name = 'ScaleSetProviderReconciliationError'; + } +} + +export class ScaleSetReconciler { + private readonly lifecycle = new Map(); + private readonly lifecycleLimit: number; + private inventory?: { expiresAt: number; value: Promise }; + + constructor( + private readonly config: ScaleSetReconcilerConfig, + private readonly serviceConfig: Pick< + ScaleSetServiceConfig, + 'sessionCloseTimeoutMs' | 'reconnectInitialBackoffMs' | 'reconnectMaxBackoffMs' + >, + private readonly dependencies: ScaleSetReconcilerDependencies, + ) { + this.lifecycleLimit = Math.min(20_000, Math.max(1000, config.maxRunners * 4)); + } + + async run(signal: AbortSignal, status: ScaleSetReconcilerStatusReporter): Promise { + let provider: ScaleSetComputeProvider; + let client: ScaleSetReconcilerClient; + try { + provider = this.dependencies.computeProviders.create(this.config.computeProvider.type, { + runnerConfigName: this.config.runnerConfigName, + scaleSetId: this.config.scaleSetId, + githubScope: this.config.githubConfigUrl, + configuration: this.config.computeProvider.configuration, + }); + const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); + client = this.dependencies.createClient(this.config, accessTokenProvider); + } catch (error) { + status.markFailed(error); + this.log('error', 'scale_set_reconciler_initialization_failed', { error }); + return; + } + + let consecutiveFailures = 0; + while (!signal.aborted) { + let session: MessageSessionClient | undefined; + let madeProgress = false; + try { + const configuredScaleSet = await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); + if ( + configuredScaleSet === null || + configuredScaleSet.id !== this.config.scaleSetId || + configuredScaleSet.name !== this.config.expectedScaleSetName || + (this.config.expectedRunnerGroupId !== undefined && + configuredScaleSet.runnerGroupId !== this.config.expectedRunnerGroupId) + ) { + throw new ScaleSetConfigurationError('configured GitHub runner scale set identity does not match'); + } + session = await client.createMessageSessionClient(this.config.scaleSetId, this.config.sessionOwner, { signal }); + status.markSessionReady(); + this.log('info', 'scale_set_session_created'); + let latestStatistics = session.session.statistics ?? undefined; + let lastMessageId = 0; + if (latestStatistics !== undefined) { + await this.reconcile(client, provider, latestStatistics, signal); + madeProgress = true; + consecutiveFailures = 0; + status.markProgress(); + } + + while (!signal.aborted) { + const message = await session.getMessage(lastMessageId, this.config.maxRunners, { signal }); + if (message === null) { + if (latestStatistics === undefined) { + throw new ScaleSetProtocolError('message session returned no message and no statistics snapshot'); + } + await this.reconcile(client, provider, latestStatistics, signal); + } else { + if (message.statistics === null) { + throw new ScaleSetProtocolError(`scale-set message ${message.messageId} contains no statistics`); + } + latestStatistics = message.statistics; + lastMessageId = message.messageId; + const requestIds = uniqueRequestIds(message); + if (requestIds.length > 0) await session.acquireJobs(requestIds, { signal }); + this.observeLifecycle(message); + await this.reconcile(client, provider, latestStatistics, signal); + // Acknowledge only after the idempotent provider reconciliation has + // succeeded. Failures intentionally leave the message for redelivery. + await session.deleteMessage(message.messageId, { signal }); + this.pruneCompletedLifecycle(message); + } + madeProgress = true; + consecutiveFailures = 0; + status.markProgress(); + } + } catch (error) { + if (signal.aborted) break; + if (isFatalReconcilerError(error)) { + status.markFailed(error); + this.log('error', 'scale_set_reconciler_failed', { error }); + return; + } + consecutiveFailures = madeProgress ? 1 : consecutiveFailures + 1; + status.markReconnecting(error); + this.log('warn', 'scale_set_reconciler_reconnecting', { consecutiveFailures, error }); + } finally { + if (session !== undefined) await this.closeSession(session); + } + + if (!signal.aborted) { + await this.dependencies.sleep( + calculateReconnectDelay( + consecutiveFailures, + this.serviceConfig.reconnectInitialBackoffMs, + this.serviceConfig.reconnectMaxBackoffMs, + this.dependencies.random, + ), + signal, + ); + } + } + status.markStopping(); + } + + private async reconcile( + client: ScaleSetReconcilerClient, + provider: ScaleSetComputeProvider, + statistics: RunnerScaleSetStatistic, + signal: AbortSignal, + ): Promise { + const desiredRunners = calculateDesiredRunners( + statistics.totalAssignedJobs, + this.config.minRunners, + this.config.maxRunners, + ); + const callbacks = { + signal, + generateJitConfiguration: async ({ + runnerName, + signal: callbackSignal, + }: { + runnerName: string; + signal?: AbortSignal; + }) => { + const jit = await client.generateJitRunnerConfig( + { name: runnerName, workFolder: this.config.workFolder }, + this.config.scaleSetId, + { signal: callbackSignal ?? signal }, + ); + if ( + jit.runner === null || + jit.runner.name !== runnerName || + jit.runner.runnerScaleSetId !== this.config.scaleSetId || + !Number.isSafeInteger(jit.runner.id) || + jit.runner.id <= 0 + ) { + throw new ScaleSetProtocolError('GitHub returned a mismatched runner identity for JIT configuration'); + } + if ( + typeof jit.encodedJITConfig !== 'string' || + jit.encodedJITConfig === '' || + Buffer.byteLength(jit.encodedJITConfig, 'utf8') > MAX_JIT_CONFIGURATION_BYTES + ) { + throw new ScaleSetProtocolError('GitHub returned an invalid JIT configuration'); + } + return { + encodedJitConfiguration: jit.encodedJITConfig, + runnerId: jit.runner.id, + runnerName: jit.runner.name, + scaleSetId: jit.runner.runnerScaleSetId, + }; + }, + removeRunner: async (expected: { + runnerId: number; + runnerName: string; + scaleSetId: number; + signal?: AbortSignal; + }) => { + const callbackSignal = expected.signal ?? signal; + const runner = await client.getRunnerByName(expected.runnerName, { signal: callbackSignal }); + if (runner === null) return { status: 'retained_unknown' as const }; + if ( + runner.id !== expected.runnerId || + runner.name !== expected.runnerName || + runner.runnerScaleSetId !== expected.scaleSetId || + expected.scaleSetId !== this.config.scaleSetId + ) { + return { status: 'retained_unknown' as const }; + } + const githubRunner = await client.getGitHubRunner(expected.runnerId, { signal: callbackSignal }); + if (githubRunner === null) return { status: 'retained_unknown' as const }; + if (githubRunner.id !== expected.runnerId || githubRunner.name !== expected.runnerName) { + return { status: 'retained_unknown' as const }; + } + if ( + typeof githubRunner.busy !== 'boolean' || + (githubRunner.status !== 'online' && githubRunner.status !== 'offline') + ) { + return { status: 'retained_unknown' as const }; + } + if (githubRunner.busy) return { status: 'retained_busy' as const }; + try { + await client.removeRunner(runner.id, { signal: callbackSignal }); + } catch (error) { + if (isScaleSetHttpError(error) && error.status === 404) return { status: 'removed' as const }; + throw error; + } + return { status: 'removed' as const }; + }, + }; + let result = await provider.reconcile({ + desiredRunners, + bootTimeoutMinutes: this.config.bootTimeoutMinutes, + runnerInventoryComplete: false, + runnerStates: this.lifecycleStates(), + ...callbacks, + }); + validateProviderResult(result, desiredRunners); + if (result.needsRunnerInventory) { + const inventory = await this.loadScaleSetInventory(client, signal); + result = await provider.reconcile({ + desiredRunners, + bootTimeoutMinutes: this.config.bootTimeoutMinutes, + runnerInventoryComplete: true, + runnerStates: this.mergeLifecycle(inventory), + ...callbacks, + }); + validateProviderResult(result, desiredRunners); + if (result.needsRunnerInventory) { + throw new ScaleSetProtocolError( + 'scale-set compute provider requested inventory after a complete inventory pass', + ); + } + } + this.log('info', 'scale_set_reconciled', { + desiredRunners, + currentRunners: result.currentRunners, + status: result.status, + actions: result.actions, + errorCount: result.errors.length, + }); + if (result.status === 'retained') { + this.log('warn', 'scale_set_capacity_retained', { + desiredRunners, + currentRunners: result.currentRunners, + retainedBusy: result.actions.retainedBusy, + retainedUnknown: result.actions.retainedUnknown, + }); + return; + } + if (result.status !== 'converged') { + throw new ScaleSetProviderReconciliationError(result, result.status === 'retryable_error'); + } + } + + private observeLifecycle(message: RunnerScaleSetMessage): void { + for (const runner of message.jobStartedMessages) + this.rememberLifecycle(runner.runnerId, runner.runnerName, 'started'); + for (const runner of message.jobCompletedMessages) { + this.rememberLifecycle(runner.runnerId, runner.runnerName, 'completed'); + } + } + + private rememberLifecycle(runnerId: number, runnerName: string, lifecycle: ScaleSetRunnerLifecycle): void { + if (!Number.isSafeInteger(runnerId) || runnerId <= 0 || runnerName === '') return; + const current = this.lifecycle.get(runnerName); + if (current !== undefined && current.runnerId !== runnerId) { + this.lifecycle.delete(runnerName); + return; + } + this.lifecycle.set(runnerName, { runnerId, runnerName, scaleSetId: this.config.scaleSetId, lifecycle }); + while (this.lifecycle.size > this.lifecycleLimit) { + const oldest = this.lifecycle.keys().next().value as string | undefined; + if (oldest === undefined) break; + this.lifecycle.delete(oldest); + } + } + + private pruneCompletedLifecycle(message: RunnerScaleSetMessage): void { + for (const runner of message.jobCompletedMessages) { + const observation = this.lifecycle.get(runner.runnerName); + if (observation?.runnerId === runner.runnerId && observation.lifecycle === 'completed') { + this.lifecycle.delete(runner.runnerName); + } + } + } + + private mergeLifecycle(inventory: readonly GitHubScaleSetRunnerState[]): ScaleSetRunnerState[] { + const result = inventory.map((runner): ScaleSetRunnerState => { + const observation = this.lifecycle.get(runner.runnerName); + const lifecycle = + observation !== undefined && + observation.runnerId === runner.runnerId && + observation.scaleSetId === runner.scaleSetId + ? observation.lifecycle + : 'unknown'; + return { ...runner, lifecycle }; + }); + const identities = new Set(result.map((runner) => `${runner.runnerId}\u0000${runner.runnerName}`)); + for (const observation of this.lifecycle.values()) { + if (identities.has(`${observation.runnerId}\u0000${observation.runnerName}`)) continue; + result.push({ ...observation, status: 'unknown', busy: undefined }); + } + return result; + } + + private lifecycleStates(): ScaleSetRunnerState[] { + return [...this.lifecycle.values()].map((observation) => ({ + ...observation, + status: 'unknown', + busy: undefined, + })); + } + + private inventoryCacheKey(): string { + const app = this.config.githubApp; + return [ + this.config.githubConfigUrl, + app.appIdParameterName, + app.installationIdParameterName, + app.privateKeyParameterName, + ].join('\u0000'); + } + + private async loadScaleSetInventory( + client: ScaleSetReconcilerClient, + signal: AbortSignal, + ): Promise { + if (this.inventory !== undefined && this.inventory.expiresAt > Date.now()) return await this.inventory.value; + const value = Promise.all([ + client.listRunners({ signal }), + this.dependencies.runnerInventory.get( + this.inventoryCacheKey(), + async () => await client.listGitHubRunners({ signal }), + ), + ]).then(([actionsRunners, githubRunners]) => + joinRunnerInventory(actionsRunners, githubRunners, this.config.scaleSetId), + ); + this.inventory = { expiresAt: Date.now() + SCALE_SET_INVENTORY_TTL_MS, value }; + try { + return await value; + } catch (error) { + if (this.inventory?.value === value) this.inventory = undefined; + throw error; + } + } + + private async closeSession(session: Pick): Promise { + try { + await session.close({ signal: this.dependencies.closeSignal(this.serviceConfig.sessionCloseTimeoutMs) }); + } catch (error) { + this.log('warn', 'scale_set_session_close_failed', { error }); + } + } + + private log(level: 'info' | 'warn' | 'error', event: string, attributes: Record = {}): void { + this.dependencies.logger[level](event, { + groupRunnerConfig: this.config.runnerConfigName, + scaleSetId: this.config.scaleSetId, + ...attributes, + }); + } +} + +export function calculateDesiredRunners(totalAssignedJobs: number, minRunners: number, maxRunners: number): number { + if (!Number.isSafeInteger(totalAssignedJobs) || totalAssignedJobs < 0) { + throw new ScaleSetProtocolError('statistics.totalAssignedJobs must be a non-negative integer'); + } + // maxRunners bounds newly requested idle capacity, but an operator reducing + // it must never make already-assigned work a scale-down target. + return Math.max(totalAssignedJobs, Math.min(maxRunners, minRunners + totalAssignedJobs)); +} + +export function calculateReconnectDelay( + attempt: number, + initialBackoffMs: number, + maxBackoffMs: number, + random: () => number = Math.random, +): number { + if (!Number.isSafeInteger(attempt) || attempt <= 0) throw new Error('attempt must be a positive integer'); + const ceiling = Math.min(maxBackoffMs, initialBackoffMs * 2 ** Math.min(attempt - 1, 30)); + const value = Math.max(0, Math.min(1, random())); + return Math.floor(ceiling / 2 + (ceiling / 2) * value); +} + +export async function abortableSleep(delayMs: number, signal: AbortSignal): Promise { + if (signal.aborted || delayMs <= 0) return; + await new Promise((resolve) => { + const done = () => { + clearTimeout(timeout); + signal.removeEventListener('abort', done); + resolve(); + }; + const timeout = setTimeout(done, delayMs); + signal.addEventListener('abort', done, { once: true }); + }); +} + +function uniqueRequestIds(message: RunnerScaleSetMessage): number[] { + return [...new Set(message.jobAvailableMessages.map(({ runnerRequestId }) => runnerRequestId))]; +} + +export function validateProviderResult(result: ScaleSetReconcileResult, desiredRunners: number): void { + const value = result as unknown; + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); + } + const record = value as Record; + const statuses = new Set(['converged', 'retained', 'retryable_error', 'non_retryable_error']); + if (!statuses.has(record.status as string)) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid status'); + } + if (record.desiredRunners !== desiredRunners || !boundedCount(record.currentRunners)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid capacity counts'); + } + if (typeof record.needsRunnerInventory !== 'boolean') { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid inventory signal'); + } + const actions = record.actions; + if (typeof actions !== 'object' || actions === null || Array.isArray(actions)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid actions'); + } + for (const key of ['launched', 'terminated', 'retainedBusy', 'retainedUnknown']) { + if (!boundedCount((actions as Record)[key])) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid action counts'); + } + } + if (!Array.isArray(record.errors) || record.errors.length > 1000) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid errors'); + } + const operations = new Set([ + 'validate', + 'reconcile', + 'list', + 'launch', + 'generate_jit_configuration', + 'publish_jit_configuration', + 'remove_runner', + 'terminate', + ]); + for (const error of record.errors) { + if (typeof error !== 'object' || error === null || Array.isArray(error)) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); + } + const metadata = error as Record; + if ( + !operations.has(metadata.operation as string) || + typeof metadata.retryable !== 'boolean' || + typeof metadata.code !== 'string' || + !/^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(metadata.code) || + !optionalBoundedMetadata(metadata.runnerName) || + !optionalBoundedMetadata(metadata.resourceId) + ) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); + } + } +} + +function boundedCount(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0 && (value as number) <= 2_147_483_647; +} + +function optionalBoundedMetadata(value: unknown): value is string | undefined { + return value === undefined || (typeof value === 'string' && value.length <= 256 && !hasAsciiControlCharacter(value)); +} + +function hasAsciiControlCharacter(value: string): boolean { + return [...value].some((character) => { + const code = character.charCodeAt(0); + return code <= 31 || code === 127; + }); +} + +function isFatalReconcilerError(error: unknown): boolean { + if (error instanceof ScaleSetConfigurationError || error instanceof ScaleSetProtocolError) return true; + if (error instanceof ScaleSetProviderReconciliationError) return !error.retryable; + if (!isScaleSetHttpError(error)) return false; + return error.status >= 400 && error.status < 500 && ![408, 409, 425, 429].includes(error.status); +} + +function joinRunnerInventory( + actionsRunners: readonly { id: number; name: string; runnerScaleSetId: number }[], + githubRunners: readonly GitHubRunnerReference[], + scaleSetId: number, +): GitHubScaleSetRunnerState[] { + const githubById = new Map(); + const duplicateIds = new Set(); + for (const runner of githubRunners) { + if (githubById.has(runner.id)) duplicateIds.add(runner.id); + else githubById.set(runner.id, runner); + } + return actionsRunners + .filter((runner) => runner.runnerScaleSetId === scaleSetId) + .map((runner) => { + const githubRunner = duplicateIds.has(runner.id) ? undefined : githubById.get(runner.id); + const exact = githubRunner?.name === runner.name; + return { + runnerId: runner.id, + runnerName: runner.name, + scaleSetId: runner.runnerScaleSetId, + status: + exact && (githubRunner.status === 'online' || githubRunner.status === 'offline') + ? githubRunner.status + : 'unknown', + busy: exact && typeof githubRunner.busy === 'boolean' ? githubRunner.busy : undefined, + }; + }); +} diff --git a/lambdas/services/scale-set/tsconfig.json b/lambdas/services/scale-set/tsconfig.json new file mode 100644 index 0000000000..714aa27b6b --- /dev/null +++ b/lambdas/services/scale-set/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.json", + "compilerOptions": { + "types": ["node", "vitest/globals"] + }, + "include": ["src/**/*"], + "exclude": ["src/**/*.test.ts"] +} diff --git a/lambdas/services/scale-set/vitest.config.ts b/lambdas/services/scale-set/vitest.config.ts new file mode 100644 index 0000000000..28a41aa2aa --- /dev/null +++ b/lambdas/services/scale-set/vitest.config.ts @@ -0,0 +1,18 @@ +import { mergeConfig } from 'vitest/config'; +import defaultConfig from '../../vitest.base.config'; + +export default mergeConfig(defaultConfig, { + root: __dirname, + test: { + coverage: { + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/index.ts', 'src/main.ts'], + thresholds: { + statements: 80, + branches: 70, + functions: 80, + lines: 80, + }, + }, + }, +}); diff --git a/lambdas/yarn.lock b/lambdas/yarn.lock index d9cd4eccc4..45b8a14249 100644 --- a/lambdas/yarn.lock +++ b/lambdas/yarn.lock @@ -138,6 +138,7 @@ __metadata: "@aws-github-runner/aws-powertools-util": "npm:*" "@aws-github-runner/aws-ssm-util": "npm:*" "@aws-sdk/client-ec2": "npm:^3.1009.0" + "@aws-sdk/client-ssm": "npm:^3.1009.0" "@octokit/rest": "npm:22.0.1" aws-sdk-client-mock: "npm:^4.1.0" aws-sdk-client-mock-jest: "npm:^4.1.0" @@ -199,6 +200,32 @@ __metadata: languageName: unknown linkType: soft +"@aws-github-runner/github-actions-scale-set@npm:*, @aws-github-runner/github-actions-scale-set@workspace:libs/github-actions-scale-set": + version: 0.0.0-use.local + resolution: "@aws-github-runner/github-actions-scale-set@workspace:libs/github-actions-scale-set" + dependencies: + "@types/node": "npm:^22.19.3" + typescript: "npm:^5.9.3" + languageName: unknown + linkType: soft + +"@aws-github-runner/scale-set-service@workspace:services/scale-set": + version: 0.0.0-use.local + resolution: "@aws-github-runner/scale-set-service@workspace:services/scale-set" + dependencies: + "@aws-github-runner/aws-ssm-util": "npm:*" + "@aws-github-runner/compute-providers": "npm:*" + "@aws-github-runner/github-actions-scale-set": "npm:*" + "@aws-sdk/client-ssm": "npm:^3.1009.0" + "@octokit/auth-app": "npm:8.2.0" + "@octokit/request": "npm:^9.2.2" + "@types/node": "npm:^22.19.3" + "@vercel/ncc": "npm:0.38.4" + typescript: "npm:^5.9.3" + undici: "npm:^6.19.2" + languageName: unknown + linkType: soft + "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher": version: 0.0.0-use.local resolution: "@aws-github-runner/termination-watcher@workspace:functions/termination-watcher" @@ -11025,6 +11052,13 @@ __metadata: languageName: node linkType: hard +"undici@npm:^6.19.2": + version: 6.28.0 + resolution: "undici@npm:6.28.0" + checksum: 10c0/3029a70df06b38b5b2f30732932a1e92544c753cd82c8abdf0d35afad48e0ba91612e79fe3a442dbbb9434d6a9eba2b714d5ea28984c903dda2b5d5444f38354 + languageName: node + linkType: hard + "unicode-canonical-property-names-ecmascript@npm:^2.0.0": version: 2.0.0 resolution: "unicode-canonical-property-names-ecmascript@npm:2.0.0" From e5f9768493b621346429fb82f2b0982fb07ed950 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 26 Aug 2026 14:42:41 +0200 Subject: [PATCH 02/19] refactor(scale-set): split EC2 provider contexts --- .../aws/ec2/src/scale-set/configuration.ts | 349 +++++ .../aws/ec2/src/scale-set/inventory.ts | 249 ++++ .../aws/ec2/src/scale-set/provider.test.ts | 37 +- .../aws/ec2/src/scale-set/provider.ts | 1166 +---------------- .../aws/ec2/src/scale-set/reconcile.ts | 171 +++ .../aws/ec2/src/scale-set/scale-down.ts | 101 ++ .../aws/ec2/src/scale-set/scale-up.ts | 255 ++++ 7 files changed, 1197 insertions(+), 1131 deletions(-) create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts new file mode 100644 index 0000000000..17fa0cce37 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts @@ -0,0 +1,349 @@ +import type { Tag as SsmTag } from '@aws-sdk/client-ssm'; + +import type { Ec2OverrideConfig, RunnerInputParameters } from '../runners.d'; +import { isRecord, NonRetryableScaleSetError } from './reconcile'; + +const SPOT_ALLOCATION_STRATEGIES = new Set([ + 'lowest-price', + 'diversified', + 'capacity-optimized', + 'capacity-optimized-prioritized', + 'price-capacity-optimized', +]); +const ON_DEMAND_ALLOCATION_STRATEGIES = new Set(['lowest-price', 'prioritized']); + +export interface Ec2ScaleSetProviderConfig { + region: string; + environment: string; + runnerNamePrefix: string; + jitConfigParameterPath: string; + subnets: string[]; + launchTemplateName: string; + ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; + ec2OverrideConfig?: Ec2OverrideConfig; + amiIdSsmParameterName?: string; + tracingEnabled?: boolean; + onDemandFailoverOnError?: string[]; + scaleErrors: string[]; + useDedicatedHost?: boolean; + ssmKmsKeyId?: string; + ssmParameterTags?: SsmTag[]; +} + +export interface CreateEc2ScaleSetProviderInput { + runnerConfigName: string; + scaleSetId: number; + githubScope: string; + configuration: Ec2ScaleSetProviderConfig; +} + +function rejectUnknownKeys(value: Record, allowedKeys: ReadonlySet, name: string): void { + const unknownKey = Object.keys(value).find((key) => !allowedKeys.has(key)); + if (unknownKey !== undefined) { + throw new NonRetryableScaleSetError(`Unsupported EC2 scale-set configuration field '${name}.${unknownKey}'`); + } +} + +function requireString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length === 0 || value.length > maximumLength || !pattern.test(value)) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requirePossiblyEmptyString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { + if (typeof value !== 'string' || value.length > maximumLength || !pattern.test(value)) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function optionalString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string | undefined { + if (value === undefined) return undefined; + return requireString(value, name, pattern, maximumLength); +} + +function optionalBoolean(value: unknown, name: string): boolean | undefined { + if (value === undefined) return undefined; + if (typeof value !== 'boolean') { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + return value; +} + +function requireStringArray( + value: unknown, + name: string, + pattern: RegExp, + maximumItemLength: number, + allowEmpty = false, +): string[] { + if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.length > 100) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + const parsed = value.map((item, index) => requireString(item, `${name}[${index}]`, pattern, maximumItemLength)); + if (new Set(parsed).size !== parsed.length) { + throw new NonRetryableScaleSetError(`EC2 scale-set configuration field '${name}' contains duplicate values`); + } + return parsed; +} + +function parseInstanceTypePriorities(value: unknown): Record | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'instanceTypePriorities'"); + } + + const result = Object.create(null) as Record; + for (const [instanceType, priority] of Object.entries(value)) { + requireString(instanceType, 'instanceTypePriorities key', /^[a-z0-9][a-z0-9.-]*$/, 64); + if (typeof priority !== 'number' || !Number.isSafeInteger(priority) || priority < 0 || priority > 1000) { + throw new NonRetryableScaleSetError( + `Invalid EC2 scale-set configuration priority for instance type '${instanceType}'`, + ); + } + result[instanceType] = priority; + } + return result; +} + +function requireSsmTagValue(value: unknown): string { + if (typeof value !== 'string' || value.length > 256) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + for (const character of value) { + const codePoint = character.codePointAt(0)!; + if (codePoint < 32 || codePoint === 127) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + } + } + return value; +} + +function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { + if (value === undefined) return undefined; + if (!isRecord(value)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2OverrideConfig'"); + } + + const supportedKeys = new Set([ + 'InstanceType', + 'MaxPrice', + 'SubnetId', + 'AvailabilityZone', + 'AvailabilityZoneId', + 'WeightedCapacity', + 'Priority', + 'ImageId', + ]); + if (Object.keys(value).some((key) => !supportedKeys.has(key))) { + throw new NonRetryableScaleSetError('EC2 scale-set configuration contains an unsupported launch override'); + } + + const weightedCapacity = value.WeightedCapacity; + const priority = value.Priority; + for (const [name, number] of [ + ['WeightedCapacity', weightedCapacity], + ['Priority', priority], + ] as const) { + if (number !== undefined && (typeof number !== 'number' || !Number.isFinite(number) || number < 0)) { + throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + } + } + + return { + InstanceType: optionalString( + value.InstanceType, + 'ec2OverrideConfig.InstanceType', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ) as Ec2OverrideConfig['InstanceType'], + MaxPrice: optionalString(value.MaxPrice, 'ec2OverrideConfig.MaxPrice', /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, 32), + SubnetId: optionalString(value.SubnetId, 'ec2OverrideConfig.SubnetId', /^subnet-[0-9a-f]+$/, 32), + AvailabilityZone: optionalString( + value.AvailabilityZone, + 'ec2OverrideConfig.AvailabilityZone', + /^[a-z]{2}(?:-[a-z0-9]+)+-\d[a-z]$/, + 64, + ), + AvailabilityZoneId: optionalString( + value.AvailabilityZoneId, + 'ec2OverrideConfig.AvailabilityZoneId', + /^[a-z0-9-]+$/, + 64, + ), + WeightedCapacity: weightedCapacity as number | undefined, + Priority: priority as number | undefined, + ImageId: optionalString(value.ImageId, 'ec2OverrideConfig.ImageId', /^ami-[0-9a-f]+$/, 32), + }; +} + +function parseSsmTags(value: unknown): SsmTag[] | undefined { + if (value === undefined) return undefined; + if (!Array.isArray(value) || value.length > 45) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + + const tags: SsmTag[] = []; + const keys = new Set(); + for (const item of value) { + if (!isRecord(item)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + } + const key = requireString(item.Key, 'ssmParameterTags.Key', /^[A-Za-z0-9_.:/=+@-]+$/, 128); + const tagValue = requireSsmTagValue(item.Value); + if (key.toLowerCase().startsWith('aws:') || keys.has(key)) { + throw new NonRetryableScaleSetError(`Invalid or duplicate SSM tag key '${key}'`); + } + keys.add(key); + tags.push({ Key: key, Value: tagValue }); + } + return tags; +} + +export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProviderConfig { + if (!isRecord(value)) { + throw new NonRetryableScaleSetError('EC2 scale-set provider configuration must be an object'); + } + rejectUnknownKeys( + value, + new Set([ + 'region', + 'environment', + 'runnerNamePrefix', + 'jitConfigParameterPath', + 'subnets', + 'launchTemplateName', + 'ec2instanceCriteria', + 'ec2OverrideConfig', + 'amiIdSsmParameterName', + 'tracingEnabled', + 'onDemandFailoverOnError', + 'scaleErrors', + 'useDedicatedHost', + 'ssmKmsKeyId', + 'ssmParameterTags', + ]), + 'configuration', + ); + if (!isRecord(value.ec2instanceCriteria)) { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2instanceCriteria'"); + } + rejectUnknownKeys( + value.ec2instanceCriteria, + new Set([ + 'instanceTypes', + 'instanceTypePriorities', + 'targetCapacityType', + 'maxSpotPrice', + 'instanceAllocationStrategy', + ]), + 'ec2instanceCriteria', + ); + + const targetCapacityType = value.ec2instanceCriteria.targetCapacityType; + if (targetCapacityType !== 'on-demand' && targetCapacityType !== 'spot') { + throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'targetCapacityType'"); + } + const instanceAllocationStrategy = requireString( + value.ec2instanceCriteria.instanceAllocationStrategy, + 'instanceAllocationStrategy', + /^[a-z-]+$/, + 64, + ) as RunnerInputParameters['ec2instanceCriteria']['instanceAllocationStrategy']; + const allowedAllocationStrategies = + targetCapacityType === 'spot' ? SPOT_ALLOCATION_STRATEGIES : ON_DEMAND_ALLOCATION_STRATEGIES; + if (!allowedAllocationStrategies.has(instanceAllocationStrategy)) { + throw new NonRetryableScaleSetError( + `Invalid allocation strategy '${instanceAllocationStrategy}' for '${targetCapacityType}' capacity`, + ); + } + + const jitConfigParameterPath = requireString( + value.jitConfigParameterPath, + 'jitConfigParameterPath', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ).replace(/\/$/, ''); + + return { + region: requireString(value.region, 'region', /^[a-z]{2}(?:-[a-z0-9]+)+-\d$/, 32), + environment: requireString(value.environment, 'environment', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128), + runnerNamePrefix: requirePossiblyEmptyString(value.runnerNamePrefix, 'runnerNamePrefix', /^[A-Za-z0-9._-]*$/, 45), + jitConfigParameterPath, + subnets: requireStringArray(value.subnets, 'subnets', /^subnet-[0-9a-f]+$/, 32), + launchTemplateName: requireString(value.launchTemplateName, 'launchTemplateName', /^[A-Za-z0-9()./_-]+$/, 128), + ec2instanceCriteria: { + instanceTypes: requireStringArray( + value.ec2instanceCriteria.instanceTypes, + 'instanceTypes', + /^[a-z0-9][a-z0-9.-]*$/, + 64, + ), + instanceTypePriorities: parseInstanceTypePriorities(value.ec2instanceCriteria.instanceTypePriorities), + targetCapacityType, + maxSpotPrice: optionalString( + value.ec2instanceCriteria.maxSpotPrice, + 'maxSpotPrice', + /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, + 32, + ), + instanceAllocationStrategy, + }, + ec2OverrideConfig: parseEc2OverrideConfig(value.ec2OverrideConfig), + amiIdSsmParameterName: optionalString( + value.amiIdSsmParameterName, + 'amiIdSsmParameterName', + /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, + 900, + ), + tracingEnabled: optionalBoolean(value.tracingEnabled, 'tracingEnabled'), + onDemandFailoverOnError: requireStringArray( + value.onDemandFailoverOnError ?? [], + 'onDemandFailoverOnError', + /^[A-Za-z0-9._-]+$/, + 128, + true, + ), + scaleErrors: requireStringArray(value.scaleErrors ?? [], 'scaleErrors', /^[A-Za-z0-9._-]+$/, 128, true), + useDedicatedHost: optionalBoolean(value.useDedicatedHost, 'useDedicatedHost'), + ssmKmsKeyId: optionalString(value.ssmKmsKeyId, 'ssmKmsKeyId', /^[A-Za-z0-9_:/+=,.@-]+$/, 2048), + ssmParameterTags: parseSsmTags(value.ssmParameterTags), + }; +} + +export function validateFactoryInput(input: CreateEc2ScaleSetProviderInput): void { + requireString(input.runnerConfigName, 'runnerConfigName', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128); + if (!Number.isSafeInteger(input.scaleSetId) || input.scaleSetId <= 0) { + throw new NonRetryableScaleSetError('scaleSetId must be a positive safe integer'); + } + validateCanonicalGitHubScope(input.githubScope); +} + +function validateCanonicalGitHubScope(value: unknown): string { + if (typeof value !== 'string' || value.length === 0 || value.length > 2048) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + let url: URL; + try { + url = new URL(value); + } catch { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + const parts = url.pathname + .replace(/^\/+|\/+$/g, '') + .split('/') + .filter(Boolean); + if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + url.pathname = `/${parts.join('/')}`; + const canonical = url.toString().replace(/\/$/, ''); + if (canonical !== value) { + throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + } + return value; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts new file mode 100644 index 0000000000..c3e6d52a14 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts @@ -0,0 +1,249 @@ +import { createHash } from 'node:crypto'; + +import { DescribeInstancesCommand, type EC2Client, type Instance, type Tag } from '@aws-sdk/client-ec2'; + +import type { ScaleSetReconcileRequest, ScaleSetRunnerState } from '../../../../scale-set'; +import type { CreateEc2ScaleSetProviderInput } from './configuration'; +import { retainUnknown, type MutableReconcileState } from './reconcile'; + +export const EC2_RUNNER_CONFIG_TAG = 'ghr:runner_config'; +export const EC2_SCALE_SET_ID_TAG = 'ghr:scale_set_id'; +export const EC2_GITHUB_SCOPE_HASH_TAG = 'ghr:github_scope_hash'; +export const EC2_SCALE_SET_STATE_TAG = 'ghr:scale_set_state'; +export const EC2_RUNNER_NAME_TAG = 'ghr:runner_name'; +export const EC2_GITHUB_RUNNER_ID_TAG = 'ghr:github_runner_id'; + +const APPLICATION_TAG = 'ghr:Application'; +const APPLICATION_VALUE = 'github-action-runner'; +const CREATED_BY_TAG = 'ghr:created_by'; +export const SCALE_SET_RUNNER_SOURCE = 'scale-set-service'; +const ENVIRONMENT_TAG = 'ghr:environment'; +export const GITHUB_RUNNER_NAME_MAX_LENGTH = 64; + +type Ec2ScaleSetState = 'provisioning' | 'publishing' | 'config-published' | 'retiring'; + +export interface OwnedEc2Runner { + instanceId: string; + launchTime?: Date; + githubRunnerId?: number; + runnerName?: string; + scaleSetState?: Ec2ScaleSetState; +} + +export function githubScopeHash(githubScope: string): string { + return createHash('sha256').update(githubScope, 'utf8').digest('hex'); +} + +export function runnerIdentityFromGitHubScope(githubScope: string): { + runnerOwner: string; + runnerType: 'Org' | 'Repo'; +} { + const pathParts = new URL(githubScope).pathname.replace(/^\/+|\/+$/g, '').split('/'); + if (pathParts.length === 2 && pathParts[0].toLowerCase() !== 'enterprises') { + return { runnerOwner: pathParts.join('/'), runnerType: 'Repo' }; + } + + // The legacy EC2 tags do not have an enterprise discriminator. They remain + // informational here; exact ownership is fenced by runner config, scale-set + // ID, and the canonical GitHub-scope hash. + return { + runnerOwner: pathParts[0].toLowerCase() === 'enterprises' ? pathParts[1] : pathParts[0], + runnerType: 'Org', + }; +} + +export function ownershipTags(input: CreateEc2ScaleSetProviderInput): Tag[] { + return [ + { Key: ENVIRONMENT_TAG, Value: input.configuration.environment }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, + ]; +} + +export async function listOwnedRunners( + input: CreateEc2ScaleSetProviderInput, + ec2Client: EC2Client, + signal: AbortSignal, +): Promise { + const runners: OwnedEc2Runner[] = []; + let nextToken: string | undefined; + do { + const response = await ec2Client.send( + new DescribeInstancesCommand({ + Filters: [ + { Name: 'instance-state-name', Values: ['pending', 'running'] }, + { Name: `tag:${APPLICATION_TAG}`, Values: [APPLICATION_VALUE] }, + { Name: `tag:${CREATED_BY_TAG}`, Values: [SCALE_SET_RUNNER_SOURCE] }, + { Name: `tag:${ENVIRONMENT_TAG}`, Values: [input.configuration.environment] }, + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: [input.runnerConfigName] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: [String(input.scaleSetId)] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash(input.githubScope)] }, + ], + NextToken: nextToken, + }), + { abortSignal: signal }, + ); + nextToken = response.NextToken; + + for (const instance of response.Reservations?.flatMap((reservation) => reservation.Instances ?? []) ?? []) { + const runner = parseOwnedRunner(instance, input); + if (runner) runners.push(runner); + } + } while (nextToken); + + return runners; +} + +function parseOwnedRunner(instance: Instance, input: CreateEc2ScaleSetProviderInput): OwnedEc2Runner | undefined { + if (!instance.InstanceId) return undefined; + const tags = new Map((instance.Tags ?? []).flatMap((tag) => (tag.Key ? [[tag.Key, tag.Value]] : []))); + + if ( + tags.get(APPLICATION_TAG) !== APPLICATION_VALUE || + tags.get(CREATED_BY_TAG) !== SCALE_SET_RUNNER_SOURCE || + tags.get(ENVIRONMENT_TAG) !== input.configuration.environment || + tags.get(EC2_RUNNER_CONFIG_TAG) !== input.runnerConfigName || + tags.get(EC2_SCALE_SET_ID_TAG) !== String(input.scaleSetId) || + tags.get(EC2_GITHUB_SCOPE_HASH_TAG) !== githubScopeHash(input.githubScope) + ) { + return undefined; + } + + const taggedRunnerId = tags.get(EC2_GITHUB_RUNNER_ID_TAG); + const githubRunnerId = taggedRunnerId === undefined ? undefined : Number(taggedRunnerId); + const rawScaleSetState = tags.get(EC2_SCALE_SET_STATE_TAG); + const scaleSetState = ['provisioning', 'publishing', 'config-published', 'retiring'].includes(rawScaleSetState ?? '') + ? (rawScaleSetState as Ec2ScaleSetState) + : undefined; + + return { + instanceId: instance.InstanceId, + launchTime: instance.LaunchTime, + githubRunnerId: Number.isSafeInteger(githubRunnerId) && githubRunnerId! > 0 ? githubRunnerId : undefined, + runnerName: tags.get(EC2_RUNNER_NAME_TAG), + scaleSetState, + }; +} + +function validRunnerState(value: ScaleSetRunnerState): boolean { + return ( + Number.isSafeInteger(value.runnerId) && + value.runnerId > 0 && + Number.isSafeInteger(value.scaleSetId) && + value.scaleSetId > 0 && + typeof value.runnerName === 'string' && + value.runnerName.length > 0 && + value.runnerName.length <= GITHUB_RUNNER_NAME_MAX_LENGTH && + ['online', 'offline', 'unknown'].includes(value.status) && + (typeof value.busy === 'boolean' || value.busy === undefined) && + ['started', 'completed', 'unknown'].includes(value.lifecycle) + ); +} + +export function indexRunnerStates( + runnerStates: readonly ScaleSetRunnerState[], + scaleSetId: number, +): { byName: Map; ambiguousNames: Set; ambiguousIds: Set } { + const byName = new Map(); + const byId = new Map(); + const ambiguousNames = new Set(); + const ambiguousIds = new Set(); + + for (const state of runnerStates) { + if (!validRunnerState(state) || state.scaleSetId !== scaleSetId) continue; + if (byName.has(state.runnerName)) ambiguousNames.add(state.runnerName); + const existingName = byId.get(state.runnerId); + if (existingName !== undefined && existingName !== state.runnerName) { + ambiguousIds.add(state.runnerId); + ambiguousNames.add(existingName); + ambiguousNames.add(state.runnerName); + } + byName.set(state.runnerName, state); + byId.set(state.runnerId, state.runnerName); + } + return { byName, ambiguousNames, ambiguousIds }; +} + +export function matchingRunnerState( + runner: OwnedEc2Runner, + index: ReturnType, + scaleSetId: number, +): ScaleSetRunnerState | undefined { + if (!runner.runnerName || !runner.githubRunnerId) return undefined; + if (index.ambiguousNames.has(runner.runnerName) || index.ambiguousIds.has(runner.githubRunnerId)) return undefined; + const state = index.byName.get(runner.runnerName); + if ( + !state || + state.runnerId !== runner.githubRunnerId || + state.runnerName !== runner.runnerName || + state.scaleSetId !== scaleSetId + ) { + return undefined; + } + return state; +} + +function isWithinBootTimeout(runner: OwnedEc2Runner, bootTimeoutMinutes: number, now: number): boolean { + const launchTime = runner.launchTime?.getTime(); + if (launchTime === undefined || !Number.isFinite(launchTime)) return false; + const ageMilliseconds = now - launchTime; + return ageMilliseconds >= 0 && ageMilliseconds < bootTimeoutMinutes * 60_000; +} + +function isConfirmedServingState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.status === 'online'; +} + +export function servingCapacity( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + now: number, +): OwnedEc2Runner[] { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const serving: OwnedEc2Runner[] = []; + + for (const runner of runners) { + if (runner.scaleSetState !== 'config-published') { + // An interrupted publication may already have been consumed. Preserve it, + // but do not let it suppress replacement capacity indefinitely. + retainUnknown(state, runner.instanceId); + continue; + } + + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + if (githubState !== undefined && isConfirmedServingState(githubState)) { + serving.push(runner); + continue; + } + if (isWithinBootTimeout(runner, request.bootTimeoutMinutes, now)) { + serving.push(runner); + continue; + } + + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) { + // Treat the stale handoff provisionally as serving until the controller + // supplies one complete joined inventory. This avoids a blind replacement + // before GitHub identity can be checked. + state.needsRunnerInventory = true; + serving.push(runner); + } + } + + return serving; +} + +export function isSafeScaleDownState(state: ScaleSetRunnerState): boolean { + return ( + (state.lifecycle === 'completed' && state.busy !== true) || + (state.lifecycle !== 'started' && state.status === 'online' && state.busy === false) + ); +} + +export function isBusyState(state: ScaleSetRunnerState): boolean { + return state.lifecycle === 'started' || state.busy === true; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts index 7e10f010d9..6fe5f27bcc 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts @@ -8,7 +8,7 @@ import { TerminateInstancesCommand, type Instance, } from '@aws-sdk/client-ec2'; -import { DeleteParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { DeleteParameterCommand, GetParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; import { mockClient } from 'aws-sdk-client-mock'; import 'aws-sdk-client-mock-jest/vitest'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -127,13 +127,15 @@ function createRequest(overrides: Partial = {}): Scale }; } -function provider(options: { githubScope?: string; now?: () => number } = {}) { +function provider( + options: { githubScope?: string; now?: () => number; configuration?: Ec2ScaleSetProviderConfig } = {}, +) { return createEc2ScaleSetProvider( { runnerConfigName: 'linux', scaleSetId: 42, githubScope: options.githubScope ?? githubScope, - configuration: config, + configuration: options.configuration ?? config, }, { ec2Client, @@ -176,13 +178,13 @@ describe('EC2 scale-set provider configuration', () => { }); it('does not expose configurable EC2 ownership or lifecycle tags', () => { - expect(Object.keys(config)).not.toContain('additionalTags'); + expect(Object.keys(config)).not.toContain('orchestrationTags'); expect(() => parseEc2ScaleSetProviderConfig({ ...config, - additionalTags: [{ Key: EC2_SCALE_SET_ID_TAG, Value: 'another-scale-set' }], + orchestrationTags: [{ Key: EC2_SCALE_SET_ID_TAG, Value: 'another-scale-set' }], }), - ).toThrow("Unsupported EC2 scale-set configuration field 'configuration.additionalTags'"); + ).toThrow("Unsupported EC2 scale-set configuration field 'configuration.orchestrationTags'"); }); it('rejects non-canonical GitHub ownership scopes before creating clients', () => { @@ -217,6 +219,9 @@ describe('EC2 scale-set reconciliation', () => { expect(result).toMatchObject({ status: 'converged', desiredRunners: 1, currentRunners: 1 }); expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { Filters: expect.arrayContaining([ + { Name: 'tag:ghr:Application', Values: ['github-action-runner'] }, + { Name: 'tag:ghr:created_by', Values: ['scale-set-service'] }, + { Name: 'tag:ghr:environment', Values: ['unit-test'] }, { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: ['linux'] }, { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: ['42'] }, { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash] }, @@ -449,6 +454,8 @@ describe('EC2 scale-set reconciliation', () => { expect.objectContaining({ ResourceType: 'instance', Tags: expect.arrayContaining([ + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:environment', Value: 'unit-test' }, { Key: 'ghr:Owner', Value: 'example' }, { Key: 'ghr:Type', Value: 'Org' }, { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, @@ -473,6 +480,24 @@ describe('EC2 scale-set reconciliation', () => { }); }); + it('resolves an AMI parameter through the provider-owned SSM client', async () => { + const instanceId = 'i-1234567890abcdef0'; + const amiIdSsmParameterName = '/github-action-runners/unit-test/ami'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(GetParameterCommand).resolves({ Parameter: { Value: 'ami-0123456789abcdef0' } }); + + const result = await provider({ + configuration: { ...config, amiIdSsmParameterName }, + }).reconcile(createRequest()); + + expect(result.actions.launched).toBe(1); + expect(ssmMock).toHaveReceivedCommandWith(GetParameterCommand, { + Name: amiIdSsmParameterName, + WithDecryption: true, + }); + }); + it('does not remove an unrelated GitHub runner when JIT identity validation fails', async () => { const instanceId = 'i-1234567890abcdef0'; ec2Mock.on(DescribeInstancesCommand).resolves({}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts index e775527127..8bfa056b17 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts @@ -1,72 +1,39 @@ -import { createHash } from 'node:crypto'; +import { EC2Client } from '@aws-sdk/client-ec2'; +import { SSMClient } from '@aws-sdk/client-ssm'; -import { CreateTagsCommand, DescribeInstancesCommand, EC2Client, type Instance, type Tag } from '@aws-sdk/client-ec2'; +import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '../../../../scale-set'; +import { createEc2RunnerClient } from '../runners'; import { - DeleteParameterCommand, - GetParameterCommand, - PutParameterCommand, - SSMClient, - type Tag as SsmTag, -} from '@aws-sdk/client-ssm'; - -import type { - GenerateScaleSetJitConfigurationResult, - ScaleSetComputeProvider, - ScaleSetReconcileActions, - ScaleSetReconcileError, - ScaleSetReconcileOperation, - ScaleSetReconcileRequest, - ScaleSetReconcileResult, - ScaleSetRunnerState, -} from '../../../../scale-set'; -import { createRunner, terminateRunner } from '../runners'; -import type { Ec2OverrideConfig, RunnerInputParameters } from '../runners.d'; - -export const EC2_RUNNER_CONFIG_TAG = 'ghr:runner_config'; -export const EC2_SCALE_SET_ID_TAG = 'ghr:scale_set_id'; -export const EC2_GITHUB_SCOPE_HASH_TAG = 'ghr:github_scope_hash'; -export const EC2_SCALE_SET_STATE_TAG = 'ghr:scale_set_state'; -export const EC2_RUNNER_NAME_TAG = 'ghr:runner_name'; -export const EC2_GITHUB_RUNNER_ID_TAG = 'ghr:github_runner_id'; + parseEc2ScaleSetProviderConfig, + validateFactoryInput, + type CreateEc2ScaleSetProviderInput, + type Ec2ScaleSetProviderConfig, +} from './configuration'; +import { listOwnedRunners, servingCapacity, type OwnedEc2Runner } from './inventory'; +import { + emptyState, + finish, + safeError, + throwIfAborted, + validateBootTimeout, + validateDesiredRunners, + validateInventorySignal, +} from './reconcile'; +import { scaleDown } from './scale-down'; +import { scaleUp } from './scale-up'; + +export type { CreateEc2ScaleSetProviderInput, Ec2ScaleSetProviderConfig } from './configuration'; +export { parseEc2ScaleSetProviderConfig } from './configuration'; +export { + EC2_GITHUB_RUNNER_ID_TAG, + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from './inventory'; -const APPLICATION_TAG = 'ghr:Application'; -const APPLICATION_VALUE = 'github-action-runner'; -const CREATED_BY_TAG = 'ghr:created_by'; -const CREATED_BY_VALUE = 'scale-set-service'; -const ENVIRONMENT_TAG = 'ghr:environment'; -const SSM_STANDARD_TIER_THRESHOLD = 4000; -const SSM_ADVANCED_TIER_MAX_BYTES = 8192; -const GITHUB_RUNNER_NAME_MAX_LENGTH = 64; const RETAINED_CAPACITY_REPLACEMENT_SURGE = 1; -const MAX_BOOT_TIMEOUT_MINUTES = 120; -const SPOT_ALLOCATION_STRATEGIES = new Set([ - 'lowest-price', - 'diversified', - 'capacity-optimized', - 'capacity-optimized-prioritized', - 'price-capacity-optimized', -]); -const ON_DEMAND_ALLOCATION_STRATEGIES = new Set(['lowest-price', 'prioritized']); - -type Ec2ScaleSetState = 'provisioning' | 'publishing' | 'config-published' | 'retiring'; - -export interface Ec2ScaleSetProviderConfig { - region: string; - environment: string; - runnerNamePrefix: string; - jitConfigParameterPath: string; - subnets: string[]; - launchTemplateName: string; - ec2instanceCriteria: RunnerInputParameters['ec2instanceCriteria']; - ec2OverrideConfig?: Ec2OverrideConfig; - amiIdSsmParameterName?: string; - tracingEnabled?: boolean; - onDemandFailoverOnError?: string[]; - scaleErrors: string[]; - useDedicatedHost?: boolean; - ssmKmsKeyId?: string; - ssmParameterTags?: SsmTag[]; -} export interface Ec2ScaleSetProviderDependencies { ec2Client?: EC2Client; @@ -74,373 +41,6 @@ export interface Ec2ScaleSetProviderDependencies { now?: () => number; } -export interface CreateEc2ScaleSetProviderInput { - runnerConfigName: string; - scaleSetId: number; - githubScope: string; - configuration: Ec2ScaleSetProviderConfig; -} - -interface OwnedEc2Runner { - instanceId: string; - launchTime?: Date; - githubRunnerId?: number; - runnerName?: string; - scaleSetState?: Ec2ScaleSetState; -} - -interface MutableReconcileState { - currentRunners: number; - needsRunnerInventory: boolean; - retainedUnknownResourceIds: Set; - actions: ScaleSetReconcileActions; - errors: ScaleSetReconcileError[]; -} - -class NonRetryableScaleSetError extends Error { - constructor(message: string) { - super(message); - this.name = 'NonRetryableScaleSetError'; - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function rejectUnknownKeys(value: Record, allowedKeys: ReadonlySet, name: string): void { - const unknownKey = Object.keys(value).find((key) => !allowedKeys.has(key)); - if (unknownKey !== undefined) { - throw new NonRetryableScaleSetError(`Unsupported EC2 scale-set configuration field '${name}.${unknownKey}'`); - } -} - -function requireString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { - if (typeof value !== 'string' || value.length === 0 || value.length > maximumLength || !pattern.test(value)) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); - } - return value; -} - -function requirePossiblyEmptyString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { - if (typeof value !== 'string' || value.length > maximumLength || !pattern.test(value)) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); - } - return value; -} - -function optionalString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string | undefined { - if (value === undefined) return undefined; - return requireString(value, name, pattern, maximumLength); -} - -function optionalBoolean(value: unknown, name: string): boolean | undefined { - if (value === undefined) return undefined; - if (typeof value !== 'boolean') { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); - } - return value; -} - -function requireStringArray( - value: unknown, - name: string, - pattern: RegExp, - maximumItemLength: number, - allowEmpty = false, -): string[] { - if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.length > 100) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); - } - const parsed = value.map((item, index) => requireString(item, `${name}[${index}]`, pattern, maximumItemLength)); - if (new Set(parsed).size !== parsed.length) { - throw new NonRetryableScaleSetError(`EC2 scale-set configuration field '${name}' contains duplicate values`); - } - return parsed; -} - -function parseInstanceTypePriorities(value: unknown): Record | undefined { - if (value === undefined) return undefined; - if (!isRecord(value)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'instanceTypePriorities'"); - } - - const result = Object.create(null) as Record; - for (const [instanceType, priority] of Object.entries(value)) { - requireString(instanceType, 'instanceTypePriorities key', /^[a-z0-9][a-z0-9.-]*$/, 64); - if (typeof priority !== 'number' || !Number.isSafeInteger(priority) || priority < 0 || priority > 1000) { - throw new NonRetryableScaleSetError( - `Invalid EC2 scale-set configuration priority for instance type '${instanceType}'`, - ); - } - result[instanceType] = priority; - } - return result; -} - -function requireSsmTagValue(value: unknown): string { - if (typeof value !== 'string' || value.length > 256) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); - } - for (const character of value) { - const codePoint = character.codePointAt(0)!; - if (codePoint < 32 || codePoint === 127) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); - } - } - return value; -} - -function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { - if (value === undefined) return undefined; - if (!isRecord(value)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2OverrideConfig'"); - } - - const supportedKeys = new Set([ - 'InstanceType', - 'MaxPrice', - 'SubnetId', - 'AvailabilityZone', - 'AvailabilityZoneId', - 'WeightedCapacity', - 'Priority', - 'ImageId', - ]); - if (Object.keys(value).some((key) => !supportedKeys.has(key))) { - throw new NonRetryableScaleSetError('EC2 scale-set configuration contains an unsupported launch override'); - } - - const weightedCapacity = value.WeightedCapacity; - const priority = value.Priority; - for (const [name, number] of [ - ['WeightedCapacity', weightedCapacity], - ['Priority', priority], - ] as const) { - if (number !== undefined && (typeof number !== 'number' || !Number.isFinite(number) || number < 0)) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); - } - } - - return { - InstanceType: optionalString( - value.InstanceType, - 'ec2OverrideConfig.InstanceType', - /^[a-z0-9][a-z0-9.-]*$/, - 64, - ) as Ec2OverrideConfig['InstanceType'], - MaxPrice: optionalString(value.MaxPrice, 'ec2OverrideConfig.MaxPrice', /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, 32), - SubnetId: optionalString(value.SubnetId, 'ec2OverrideConfig.SubnetId', /^subnet-[0-9a-f]+$/, 32), - AvailabilityZone: optionalString( - value.AvailabilityZone, - 'ec2OverrideConfig.AvailabilityZone', - /^[a-z]{2}(?:-[a-z0-9]+)+-\d[a-z]$/, - 64, - ), - AvailabilityZoneId: optionalString( - value.AvailabilityZoneId, - 'ec2OverrideConfig.AvailabilityZoneId', - /^[a-z0-9-]+$/, - 64, - ), - WeightedCapacity: weightedCapacity as number | undefined, - Priority: priority as number | undefined, - ImageId: optionalString(value.ImageId, 'ec2OverrideConfig.ImageId', /^ami-[0-9a-f]+$/, 32), - }; -} - -function parseSsmTags(value: unknown): SsmTag[] | undefined { - if (value === undefined) return undefined; - if (!Array.isArray(value) || value.length > 45) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); - } - - const tags: SsmTag[] = []; - const keys = new Set(); - for (const item of value) { - if (!isRecord(item)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); - } - const key = requireString(item.Key, 'ssmParameterTags.Key', /^[A-Za-z0-9_.:/=+@-]+$/, 128); - const tagValue = requireSsmTagValue(item.Value); - if (key.toLowerCase().startsWith('aws:') || keys.has(key)) { - throw new NonRetryableScaleSetError(`Invalid or duplicate SSM tag key '${key}'`); - } - keys.add(key); - tags.push({ Key: key, Value: tagValue }); - } - return tags; -} - -export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProviderConfig { - if (!isRecord(value)) { - throw new NonRetryableScaleSetError('EC2 scale-set provider configuration must be an object'); - } - rejectUnknownKeys( - value, - new Set([ - 'region', - 'environment', - 'runnerNamePrefix', - 'jitConfigParameterPath', - 'subnets', - 'launchTemplateName', - 'ec2instanceCriteria', - 'ec2OverrideConfig', - 'amiIdSsmParameterName', - 'tracingEnabled', - 'onDemandFailoverOnError', - 'scaleErrors', - 'useDedicatedHost', - 'ssmKmsKeyId', - 'ssmParameterTags', - ]), - 'configuration', - ); - if (!isRecord(value.ec2instanceCriteria)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2instanceCriteria'"); - } - rejectUnknownKeys( - value.ec2instanceCriteria, - new Set([ - 'instanceTypes', - 'instanceTypePriorities', - 'targetCapacityType', - 'maxSpotPrice', - 'instanceAllocationStrategy', - ]), - 'ec2instanceCriteria', - ); - - const targetCapacityType = value.ec2instanceCriteria.targetCapacityType; - if (targetCapacityType !== 'on-demand' && targetCapacityType !== 'spot') { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'targetCapacityType'"); - } - const instanceAllocationStrategy = requireString( - value.ec2instanceCriteria.instanceAllocationStrategy, - 'instanceAllocationStrategy', - /^[a-z-]+$/, - 64, - ) as RunnerInputParameters['ec2instanceCriteria']['instanceAllocationStrategy']; - const allowedAllocationStrategies = - targetCapacityType === 'spot' ? SPOT_ALLOCATION_STRATEGIES : ON_DEMAND_ALLOCATION_STRATEGIES; - if (!allowedAllocationStrategies.has(instanceAllocationStrategy)) { - throw new NonRetryableScaleSetError( - `Invalid allocation strategy '${instanceAllocationStrategy}' for '${targetCapacityType}' capacity`, - ); - } - - const jitConfigParameterPath = requireString( - value.jitConfigParameterPath, - 'jitConfigParameterPath', - /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, - 900, - ).replace(/\/$/, ''); - - return { - region: requireString(value.region, 'region', /^[a-z]{2}(?:-[a-z0-9]+)+-\d$/, 32), - environment: requireString(value.environment, 'environment', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128), - runnerNamePrefix: requirePossiblyEmptyString(value.runnerNamePrefix, 'runnerNamePrefix', /^[A-Za-z0-9._-]*$/, 45), - jitConfigParameterPath, - subnets: requireStringArray(value.subnets, 'subnets', /^subnet-[0-9a-f]+$/, 32), - launchTemplateName: requireString(value.launchTemplateName, 'launchTemplateName', /^[A-Za-z0-9()./_-]+$/, 128), - ec2instanceCriteria: { - instanceTypes: requireStringArray( - value.ec2instanceCriteria.instanceTypes, - 'instanceTypes', - /^[a-z0-9][a-z0-9.-]*$/, - 64, - ), - instanceTypePriorities: parseInstanceTypePriorities(value.ec2instanceCriteria.instanceTypePriorities), - targetCapacityType, - maxSpotPrice: optionalString( - value.ec2instanceCriteria.maxSpotPrice, - 'maxSpotPrice', - /^(?:0|[1-9]\d*)(?:\.\d{1,6})?$/, - 32, - ), - instanceAllocationStrategy, - }, - ec2OverrideConfig: parseEc2OverrideConfig(value.ec2OverrideConfig), - amiIdSsmParameterName: optionalString( - value.amiIdSsmParameterName, - 'amiIdSsmParameterName', - /^\/(?!.*\/\/)[A-Za-z0-9_.\-/]+$/, - 900, - ), - tracingEnabled: optionalBoolean(value.tracingEnabled, 'tracingEnabled'), - onDemandFailoverOnError: requireStringArray( - value.onDemandFailoverOnError ?? [], - 'onDemandFailoverOnError', - /^[A-Za-z0-9._-]+$/, - 128, - true, - ), - scaleErrors: requireStringArray(value.scaleErrors ?? [], 'scaleErrors', /^[A-Za-z0-9._-]+$/, 128, true), - useDedicatedHost: optionalBoolean(value.useDedicatedHost, 'useDedicatedHost'), - ssmKmsKeyId: optionalString(value.ssmKmsKeyId, 'ssmKmsKeyId', /^[A-Za-z0-9_:/+=,.@-]+$/, 2048), - ssmParameterTags: parseSsmTags(value.ssmParameterTags), - }; -} - -function validateFactoryInput(input: CreateEc2ScaleSetProviderInput): void { - requireString(input.runnerConfigName, 'runnerConfigName', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128); - if (!Number.isSafeInteger(input.scaleSetId) || input.scaleSetId <= 0) { - throw new NonRetryableScaleSetError('scaleSetId must be a positive safe integer'); - } - validateCanonicalGitHubScope(input.githubScope); -} - -function validateCanonicalGitHubScope(value: unknown): string { - if (typeof value !== 'string' || value.length === 0 || value.length > 2048) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); - } - let url: URL; - try { - url = new URL(value); - } catch { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); - } - if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); - } - const parts = url.pathname - .replace(/^\/+|\/+$/g, '') - .split('/') - .filter(Boolean); - if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); - } - url.pathname = `/${parts.join('/')}`; - const canonical = url.toString().replace(/\/$/, ''); - if (canonical !== value) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); - } - return value; -} - -function githubScopeHash(githubScope: string): string { - return createHash('sha256').update(githubScope, 'utf8').digest('hex'); -} - -function runnerIdentityFromGitHubScope(githubScope: string): { - runnerOwner: string; - runnerType: 'Org' | 'Repo'; -} { - const pathParts = new URL(githubScope).pathname.replace(/^\/+|\/+$/g, '').split('/'); - if (pathParts.length === 2 && pathParts[0].toLowerCase() !== 'enterprises') { - return { runnerOwner: pathParts.join('/'), runnerType: 'Repo' }; - } - - // The legacy EC2 tags do not have an enterprise discriminator. They remain - // informational here; exact ownership is fenced by runner config, scale-set - // ID, and the canonical GitHub-scope hash. - return { - runnerOwner: pathParts[0].toLowerCase() === 'enterprises' ? pathParts[1] : pathParts[0], - runnerType: 'Org', - }; -} - function createClients(config: Ec2ScaleSetProviderConfig, dependencies: Ec2ScaleSetProviderDependencies) { return { ec2Client: dependencies.ec2Client ?? new EC2Client({ region: config.region }), @@ -454,692 +54,6 @@ function createClients(config: Ec2ScaleSetProviderConfig, dependencies: Ec2Scale }; } -function safeError( - operation: ScaleSetReconcileOperation, - error: unknown, - details: Pick = {}, -): ScaleSetReconcileError { - return { - operation, - code: safeErrorCode(error), - retryable: isRetryableError(error), - ...details, - }; -} - -function safeErrorCode(error: unknown): string { - if (error instanceof NonRetryableScaleSetError) return 'INVALID_CONFIGURATION'; - if (!isRecord(error)) return 'UNEXPECTED_ERROR'; - for (const candidate of [error.name, error.code]) { - if (typeof candidate === 'string' && /^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(candidate)) { - return candidate; - } - } - return 'UNEXPECTED_ERROR'; -} - -function isRetryableError(error: unknown): boolean { - if (error instanceof NonRetryableScaleSetError) return false; - if (!isRecord(error)) return true; - - const identity = [error.name, error.code] - .filter((candidate): candidate is string => typeof candidate === 'string') - .join(' ') - .toLowerCase(); - if (/accessdenied|unauthor|forbidden|permission|validation|invalid|malformed|unsupported/.test(identity)) { - return false; - } - if (/throttl|timeout|temporar|serviceunavailable|internalserver|network|econn|socket|slowdown/.test(identity)) { - return true; - } - - const metadata = isRecord(error.$metadata) ? error.$metadata : undefined; - const status = [error.status, error.statusCode, metadata?.httpStatusCode].find( - (candidate): candidate is number => typeof candidate === 'number', - ); - if (status !== undefined) { - return status >= 500 || [408, 409, 425, 429].includes(status); - } - return true; -} - -function throwIfAborted(signal: AbortSignal, error?: unknown): void { - if (signal.aborted || (isRecord(error) && error.name === 'AbortError')) { - signal.throwIfAborted(); - throw error; - } -} - -function resultStatus( - errors: readonly ScaleSetReconcileError[], - current: number, - desired: number, - needsRunnerInventory: boolean, -) { - if (errors.some((error) => !error.retryable)) return 'non_retryable_error' as const; - if (errors.length > 0 || current < desired) return 'retryable_error' as const; - if (needsRunnerInventory) return 'retained' as const; - if (current > desired) return 'retained' as const; - return 'converged' as const; -} - -function finish(state: MutableReconcileState, desiredRunners: number): ScaleSetReconcileResult { - if (desiredRunners >= 0 && state.currentRunners < desiredRunners && state.errors.length === 0) { - state.errors.push({ - operation: 'reconcile', - code: 'CAPACITY_NOT_PROVISIONED', - retryable: true, - }); - } - return { - status: resultStatus(state.errors, state.currentRunners, desiredRunners, state.needsRunnerInventory), - desiredRunners, - currentRunners: state.currentRunners, - needsRunnerInventory: state.needsRunnerInventory, - actions: state.actions, - errors: state.errors, - }; -} - -function emptyState(currentRunners: number): MutableReconcileState { - return { - currentRunners, - needsRunnerInventory: false, - retainedUnknownResourceIds: new Set(), - actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, - errors: [], - }; -} - -function retainUnknown(state: MutableReconcileState, resourceId?: string): void { - if (resourceId === undefined) { - state.actions.retainedUnknown++; - return; - } - if (state.retainedUnknownResourceIds.has(resourceId)) return; - state.retainedUnknownResourceIds.add(resourceId); - state.actions.retainedUnknown++; -} - -function validateDesiredRunners(desiredRunners: number): ScaleSetReconcileError | undefined { - if (!Number.isSafeInteger(desiredRunners) || desiredRunners < 0 || desiredRunners > 10000) { - return { - operation: 'validate', - code: 'INVALID_DESIRED_RUNNER_COUNT', - retryable: false, - }; - } - return undefined; -} - -function validateBootTimeout(bootTimeoutMinutes: number): ScaleSetReconcileError | undefined { - if ( - !Number.isSafeInteger(bootTimeoutMinutes) || - bootTimeoutMinutes < 1 || - bootTimeoutMinutes > MAX_BOOT_TIMEOUT_MINUTES - ) { - return { - operation: 'validate', - code: 'INVALID_BOOT_TIMEOUT', - retryable: false, - }; - } - return undefined; -} - -function validateInventorySignal(runnerInventoryComplete: unknown): ScaleSetReconcileError | undefined { - if (typeof runnerInventoryComplete !== 'boolean') { - return { - operation: 'validate', - code: 'INVALID_RUNNER_INVENTORY_SIGNAL', - retryable: false, - }; - } - return undefined; -} - -function ownershipTags(input: CreateEc2ScaleSetProviderInput): Tag[] { - return [ - { Key: ENVIRONMENT_TAG, Value: input.configuration.environment }, - { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, - { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, - { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, - { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, - ]; -} - -async function listOwnedRunners( - input: CreateEc2ScaleSetProviderInput, - ec2Client: EC2Client, - signal: AbortSignal, -): Promise { - const runners: OwnedEc2Runner[] = []; - let nextToken: string | undefined; - do { - const response = await ec2Client.send( - new DescribeInstancesCommand({ - Filters: [ - { Name: 'instance-state-name', Values: ['pending', 'running'] }, - { Name: `tag:${APPLICATION_TAG}`, Values: [APPLICATION_VALUE] }, - { Name: `tag:${CREATED_BY_TAG}`, Values: [CREATED_BY_VALUE] }, - { Name: `tag:${ENVIRONMENT_TAG}`, Values: [input.configuration.environment] }, - { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: [input.runnerConfigName] }, - { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: [String(input.scaleSetId)] }, - { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash(input.githubScope)] }, - ], - NextToken: nextToken, - }), - { abortSignal: signal }, - ); - nextToken = response.NextToken; - - for (const instance of response.Reservations?.flatMap((reservation) => reservation.Instances ?? []) ?? []) { - const runner = parseOwnedRunner(instance, input); - if (runner) runners.push(runner); - } - } while (nextToken); - - return runners; -} - -function parseOwnedRunner(instance: Instance, input: CreateEc2ScaleSetProviderInput): OwnedEc2Runner | undefined { - if (!instance.InstanceId) return undefined; - const tags = new Map((instance.Tags ?? []).flatMap((tag) => (tag.Key ? [[tag.Key, tag.Value]] : []))); - - if ( - tags.get(APPLICATION_TAG) !== APPLICATION_VALUE || - tags.get(CREATED_BY_TAG) !== CREATED_BY_VALUE || - tags.get(ENVIRONMENT_TAG) !== input.configuration.environment || - tags.get(EC2_RUNNER_CONFIG_TAG) !== input.runnerConfigName || - tags.get(EC2_SCALE_SET_ID_TAG) !== String(input.scaleSetId) || - tags.get(EC2_GITHUB_SCOPE_HASH_TAG) !== githubScopeHash(input.githubScope) - ) { - return undefined; - } - - const taggedRunnerId = tags.get(EC2_GITHUB_RUNNER_ID_TAG); - const githubRunnerId = taggedRunnerId === undefined ? undefined : Number(taggedRunnerId); - const rawScaleSetState = tags.get(EC2_SCALE_SET_STATE_TAG); - const scaleSetState = ['provisioning', 'publishing', 'config-published', 'retiring'].includes(rawScaleSetState ?? '') - ? (rawScaleSetState as Ec2ScaleSetState) - : undefined; - - return { - instanceId: instance.InstanceId, - launchTime: instance.LaunchTime, - githubRunnerId: Number.isSafeInteger(githubRunnerId) && githubRunnerId! > 0 ? githubRunnerId : undefined, - runnerName: tags.get(EC2_RUNNER_NAME_TAG), - scaleSetState, - }; -} - -function validRunnerState(value: ScaleSetRunnerState): boolean { - return ( - Number.isSafeInteger(value.runnerId) && - value.runnerId > 0 && - Number.isSafeInteger(value.scaleSetId) && - value.scaleSetId > 0 && - typeof value.runnerName === 'string' && - value.runnerName.length > 0 && - value.runnerName.length <= GITHUB_RUNNER_NAME_MAX_LENGTH && - ['online', 'offline', 'unknown'].includes(value.status) && - (typeof value.busy === 'boolean' || value.busy === undefined) && - ['started', 'completed', 'unknown'].includes(value.lifecycle) - ); -} - -function indexRunnerStates( - runnerStates: readonly ScaleSetRunnerState[], - scaleSetId: number, -): { byName: Map; ambiguousNames: Set; ambiguousIds: Set } { - const byName = new Map(); - const byId = new Map(); - const ambiguousNames = new Set(); - const ambiguousIds = new Set(); - - for (const state of runnerStates) { - if (!validRunnerState(state) || state.scaleSetId !== scaleSetId) continue; - if (byName.has(state.runnerName)) ambiguousNames.add(state.runnerName); - const existingName = byId.get(state.runnerId); - if (existingName !== undefined && existingName !== state.runnerName) { - ambiguousIds.add(state.runnerId); - ambiguousNames.add(existingName); - ambiguousNames.add(state.runnerName); - } - byName.set(state.runnerName, state); - byId.set(state.runnerId, state.runnerName); - } - return { byName, ambiguousNames, ambiguousIds }; -} - -function matchingRunnerState( - runner: OwnedEc2Runner, - index: ReturnType, - scaleSetId: number, -): ScaleSetRunnerState | undefined { - if (!runner.runnerName || !runner.githubRunnerId) return undefined; - if (index.ambiguousNames.has(runner.runnerName) || index.ambiguousIds.has(runner.githubRunnerId)) return undefined; - const state = index.byName.get(runner.runnerName); - if ( - !state || - state.runnerId !== runner.githubRunnerId || - state.runnerName !== runner.runnerName || - state.scaleSetId !== scaleSetId - ) { - return undefined; - } - return state; -} - -function isWithinBootTimeout(runner: OwnedEc2Runner, bootTimeoutMinutes: number, now: number): boolean { - const launchTime = runner.launchTime?.getTime(); - if (launchTime === undefined || !Number.isFinite(launchTime)) return false; - const ageMilliseconds = now - launchTime; - return ageMilliseconds >= 0 && ageMilliseconds < bootTimeoutMinutes * 60_000; -} - -function isConfirmedServingState(state: ScaleSetRunnerState): boolean { - return state.lifecycle === 'started' || state.status === 'online'; -} - -function servingCapacity( - input: CreateEc2ScaleSetProviderInput, - runners: readonly OwnedEc2Runner[], - request: ScaleSetReconcileRequest, - state: MutableReconcileState, - now: number, -): OwnedEc2Runner[] { - const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); - const serving: OwnedEc2Runner[] = []; - - for (const runner of runners) { - if (runner.scaleSetState !== 'config-published') { - // An interrupted publication may already have been consumed. Preserve it, - // but do not let it suppress replacement capacity indefinitely. - retainUnknown(state, runner.instanceId); - continue; - } - - const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); - if (githubState !== undefined && isConfirmedServingState(githubState)) { - serving.push(runner); - continue; - } - if (isWithinBootTimeout(runner, request.bootTimeoutMinutes, now)) { - serving.push(runner); - continue; - } - - retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) { - // Treat the stale handoff provisionally as serving until the controller - // supplies one complete joined inventory. This avoids a blind replacement - // before GitHub identity can be checked. - state.needsRunnerInventory = true; - serving.push(runner); - } - } - - return serving; -} - -function isSafeScaleDownState(state: ScaleSetRunnerState): boolean { - return ( - (state.lifecycle === 'completed' && state.busy !== true) || - (state.lifecycle !== 'started' && state.status === 'online' && state.busy === false) - ); -} - -function isBusyState(state: ScaleSetRunnerState): boolean { - return state.lifecycle === 'started' || state.busy === true; -} - -async function tagRunner(instanceId: string, tags: Tag[], ec2Client: EC2Client, signal: AbortSignal): Promise { - await ec2Client.send(new CreateTagsCommand({ Resources: [instanceId], Tags: tags }), { abortSignal: signal }); -} - -async function getParameter(name: string, ssmClient: SSMClient, signal: AbortSignal): Promise { - const response = await ssmClient.send(new GetParameterCommand({ Name: name, WithDecryption: true }), { - abortSignal: signal, - }); - if (!response.Parameter?.Value) { - throw new NonRetryableScaleSetError(`AMI parameter '${name}' has no value`); - } - return response.Parameter.Value; -} - -function jitParameterName(config: Ec2ScaleSetProviderConfig, instanceId: string): string { - return `${config.jitConfigParameterPath}/${instanceId}`; -} - -function jitParameterTags(input: CreateEc2ScaleSetProviderInput, instanceId: string): SsmTag[] { - const reserved = new Set(['InstanceId', EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG, EC2_GITHUB_SCOPE_HASH_TAG]); - return [ - ...(input.configuration.ssmParameterTags ?? []).filter((tag) => tag.Key && !reserved.has(tag.Key)), - { Key: 'InstanceId', Value: instanceId }, - { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, - { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, - { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, - ]; -} - -async function publishJitConfiguration( - input: CreateEc2ScaleSetProviderInput, - instanceId: string, - encodedJitConfiguration: string, - ssmClient: SSMClient, - signal: AbortSignal, -): Promise { - const valueSize = Buffer.byteLength(encodedJitConfiguration, 'utf8'); - if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { - throw new NonRetryableScaleSetError('JIT configuration must be between 1 and 8192 bytes'); - } - - await ssmClient.send( - new PutParameterCommand({ - Name: jitParameterName(input.configuration, instanceId), - Value: encodedJitConfiguration, - Type: 'SecureString', - KeyId: input.configuration.ssmKmsKeyId, - Overwrite: false, - Tier: valueSize >= SSM_STANDARD_TIER_THRESHOLD ? 'Advanced' : 'Standard', - Tags: jitParameterTags(input, instanceId), - }), - { abortSignal: signal }, - ); -} - -async function bestEffortCancelJitPublication( - input: CreateEc2ScaleSetProviderInput, - instanceId: string, - ssmClient: SSMClient, - signal: AbortSignal, -): Promise { - try { - await ssmClient.send(new DeleteParameterCommand({ Name: jitParameterName(input.configuration, instanceId) }), { - abortSignal: signal, - }); - } catch (error) { - throwIfAborted(signal, error); - } -} - -function validateJitResult( - result: GenerateScaleSetJitConfigurationResult, - expectedRunnerName: string, - scaleSetId: number, -): void { - if ( - !Number.isSafeInteger(result.runnerId) || - result.runnerId <= 0 || - result.runnerName !== expectedRunnerName || - result.scaleSetId !== scaleSetId - ) { - throw new NonRetryableScaleSetError('JIT configuration returned an unexpected runner identity'); - } - const valueSize = Buffer.byteLength(result.encodedJitConfiguration, 'utf8'); - if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { - throw new NonRetryableScaleSetError('JIT configuration has an invalid size'); - } -} - -async function terminateUnpublishedRunner( - instanceId: string, - state: MutableReconcileState, - ec2Client: EC2Client, - signal: AbortSignal, -): Promise { - try { - await terminateRunner(instanceId, { ec2Client, signal }); - state.currentRunners--; - state.actions.terminated++; - } catch (error) { - throwIfAborted(signal, error); - retainUnknown(state, instanceId); - state.errors.push(safeError('terminate', error, { resourceId: instanceId })); - } -} - -async function cleanGitHubRunner( - jit: GenerateScaleSetJitConfigurationResult, - request: ScaleSetReconcileRequest, - state: MutableReconcileState, -): Promise { - try { - await request.removeRunner({ - runnerId: jit.runnerId, - runnerName: jit.runnerName, - scaleSetId: jit.scaleSetId, - signal: request.signal, - }); - } catch (error) { - throwIfAborted(request.signal, error); - state.errors.push(safeError('remove_runner', error, { runnerName: jit.runnerName })); - } -} - -async function configureLaunchedRunner( - input: CreateEc2ScaleSetProviderInput, - instanceId: string, - request: ScaleSetReconcileRequest, - state: MutableReconcileState, - clients: ReturnType, -): Promise { - const runnerName = `${input.configuration.runnerNamePrefix}${instanceId}`; - if (runnerName.length > GITHUB_RUNNER_NAME_MAX_LENGTH) { - state.errors.push({ - operation: 'generate_jit_configuration', - code: 'RUNNER_NAME_TOO_LONG', - retryable: false, - resourceId: instanceId, - }); - await terminateUnpublishedRunner(instanceId, state, clients.ec2Client, request.signal); - return; - } - - let jit: GenerateScaleSetJitConfigurationResult; - try { - jit = await request.generateJitConfiguration({ runnerName, signal: request.signal }); - validateJitResult(jit, runnerName, input.scaleSetId); - } catch (error) { - throwIfAborted(request.signal, error); - state.errors.push(safeError('generate_jit_configuration', error, { runnerName, resourceId: instanceId })); - await terminateUnpublishedRunner(instanceId, state, clients.ec2Client, request.signal); - return; - } - - try { - await tagRunner( - instanceId, - [ - { Key: EC2_RUNNER_NAME_TAG, Value: jit.runnerName }, - { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(jit.runnerId) }, - { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, - ], - clients.ec2Client, - request.signal, - ); - } catch (error) { - throwIfAborted(request.signal, error); - state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); - await cleanGitHubRunner(jit, request, state); - await terminateUnpublishedRunner(instanceId, state, clients.ec2Client, request.signal); - return; - } - - try { - await publishJitConfiguration(input, instanceId, jit.encodedJitConfiguration, clients.ssmClient, request.signal); - } catch (error) { - throwIfAborted(request.signal, error); - state.errors.push(safeError('publish_jit_configuration', error, { runnerName, resourceId: instanceId })); - await bestEffortCancelJitPublication(input, instanceId, clients.ssmClient, request.signal); - // Main's bootstrap reads before deleting. Even a successful controller-side - // DeleteParameter can race after that read and cannot prove non-consumption. - // Preserve both GitHub and compute state until an exact lifecycle signal is observed. - retainUnknown(state, instanceId); - return; - } - - state.actions.launched++; - try { - await tagRunner( - instanceId, - [{ Key: EC2_SCALE_SET_STATE_TAG, Value: 'config-published' }], - clients.ec2Client, - request.signal, - ); - } catch (error) { - throwIfAborted(request.signal, error); - // Publication may already have been consumed. Preserve the instance and exact GitHub identity. - retainUnknown(state, instanceId); - state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); - } -} - -async function scaleUp( - input: CreateEc2ScaleSetProviderInput, - count: number, - request: ScaleSetReconcileRequest, - state: MutableReconcileState, - clients: ReturnType, -): Promise { - const runnerIdentity = runnerIdentityFromGitHubScope(input.githubScope); - let createResult; - try { - createResult = await createRunner( - { - environment: input.configuration.environment, - runnerOwner: runnerIdentity.runnerOwner, - runnerType: runnerIdentity.runnerType, - subnets: input.configuration.subnets, - launchTemplateName: input.configuration.launchTemplateName, - ec2instanceCriteria: input.configuration.ec2instanceCriteria, - ec2OverrideConfig: input.configuration.ec2OverrideConfig, - numberOfRunners: count, - source: CREATED_BY_VALUE, - amiIdSsmParameterName: input.configuration.amiIdSsmParameterName, - tracingEnabled: input.configuration.tracingEnabled, - onDemandFailoverOnError: input.configuration.onDemandFailoverOnError, - scaleErrors: input.configuration.scaleErrors, - useDedicatedHost: input.configuration.useDedicatedHost, - additionalTags: ownershipTags(input), - }, - { - ec2Client: clients.ec2Client, - getParameter: (name) => getParameter(name, clients.ssmClient, request.signal), - signal: request.signal, - }, - ); - } catch (error) { - throwIfAborted(request.signal, error); - state.errors.push(safeError('launch', error)); - return; - } - - state.currentRunners += createResult.instances.length; - if (createResult.retryableErrorCount > 0) { - state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_RETRYABLE', retryable: true }); - } - if (createResult.nonRetryableErrorCount > 0) { - state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_NON_RETRYABLE', retryable: false }); - } - - for (const instanceId of createResult.instances) { - request.signal.throwIfAborted(); - await configureLaunchedRunner(input, instanceId, request, state, clients); - } -} - -async function terminateKnownIdleRunner( - runner: OwnedEc2Runner, - githubState: ScaleSetRunnerState, - request: ScaleSetReconcileRequest, - state: MutableReconcileState, - clients: ReturnType, -): Promise { - let removalResult; - try { - removalResult = await request.removeRunner({ - runnerId: githubState.runnerId, - runnerName: githubState.runnerName, - scaleSetId: githubState.scaleSetId, - signal: request.signal, - }); - } catch (error) { - throwIfAborted(request.signal, error); - state.errors.push( - safeError('remove_runner', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), - ); - retainUnknown(state, runner.instanceId); - return false; - } - - if (removalResult.status === 'retained_busy') { - state.actions.retainedBusy++; - return false; - } - if (removalResult.status !== 'removed') { - retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; - return false; - } - - try { - await terminateRunner(runner.instanceId, { ec2Client: clients.ec2Client, signal: request.signal }); - state.currentRunners--; - state.actions.terminated++; - return true; - } catch (error) { - throwIfAborted(request.signal, error); - state.errors.push( - safeError('terminate', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), - ); - retainUnknown(state, runner.instanceId); - return false; - } -} - -async function scaleDown( - input: CreateEc2ScaleSetProviderInput, - runners: readonly OwnedEc2Runner[], - count: number, - request: ScaleSetReconcileRequest, - state: MutableReconcileState, - clients: ReturnType, -): Promise { - const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); - const candidates: { runner: OwnedEc2Runner; githubState: ScaleSetRunnerState }[] = []; - - for (const runner of runners) { - const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); - if (!githubState) { - retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; - } else if (isBusyState(githubState)) { - state.actions.retainedBusy++; - } else if (isSafeScaleDownState(githubState)) { - candidates.push({ runner, githubState }); - } else { - retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; - } - } - - candidates.sort((left, right) => { - const launchOrder = (right.runner.launchTime?.getTime() ?? 0) - (left.runner.launchTime?.getTime() ?? 0); - return launchOrder || left.runner.instanceId.localeCompare(right.runner.instanceId); - }); - - let remaining = count; - for (const candidate of candidates) { - if (remaining === 0) break; - request.signal.throwIfAborted(); - if (await terminateKnownIdleRunner(candidate.runner, candidate.githubState, request, state, clients)) { - remaining--; - } - } -} - export function createEc2ScaleSetProvider( input: CreateEc2ScaleSetProviderInput, dependencies: Ec2ScaleSetProviderDependencies = {}, @@ -1150,6 +64,7 @@ export function createEc2ScaleSetProvider( }; validateFactoryInput(normalizedInput); const clients = createClients(normalizedInput.configuration, dependencies); + const runnerClient = createEc2RunnerClient(clients.ec2Client, clients.ssmClient); const now = dependencies.now ?? Date.now; return { @@ -1165,9 +80,10 @@ export function createEc2ScaleSetProvider( return finish(state, request.desiredRunners); } - let runners: OwnedEc2Runner[]; + const runnerOperations = runnerClient.forRequest({ signal: request.signal }); + let ownedRunners: OwnedEc2Runner[]; try { - runners = await listOwnedRunners(normalizedInput, clients.ec2Client, request.signal); + ownedRunners = await listOwnedRunners(normalizedInput, clients.ec2Client, request.signal); } catch (error) { throwIfAborted(request.signal, error); const state = emptyState(0); @@ -1175,18 +91,18 @@ export function createEc2ScaleSetProvider( return finish(state, request.desiredRunners); } - const state = emptyState(runners.length); - const servingRunners = servingCapacity(normalizedInput, runners, request, state, now()); + const state = emptyState(ownedRunners.length); + const servingRunners = servingCapacity(normalizedInput, ownedRunners, request, state, now()); if (servingRunners.length < request.desiredRunners) { const capacityDeficit = request.desiredRunners - servingRunners.length; const availableReplacementSlots = Math.max( 0, - request.desiredRunners + RETAINED_CAPACITY_REPLACEMENT_SURGE - runners.length, + request.desiredRunners + RETAINED_CAPACITY_REPLACEMENT_SURGE - ownedRunners.length, ); const launchCount = Math.min(capacityDeficit, availableReplacementSlots); if (launchCount > 0) { - await scaleUp(normalizedInput, launchCount, request, state, clients); + await scaleUp(normalizedInput, launchCount, request, state, runnerOperations, clients.ssmClient); } } else if (servingRunners.length > request.desiredRunners) { await scaleDown( @@ -1195,7 +111,7 @@ export function createEc2ScaleSetProvider( servingRunners.length - request.desiredRunners, request, state, - clients, + runnerOperations, ); } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts new file mode 100644 index 0000000000..4a94a6e842 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts @@ -0,0 +1,171 @@ +import type { + ScaleSetReconcileActions, + ScaleSetReconcileError, + ScaleSetReconcileOperation, + ScaleSetReconcileResult, +} from '../../../../scale-set'; + +const MAX_BOOT_TIMEOUT_MINUTES = 120; + +export interface MutableReconcileState { + currentRunners: number; + needsRunnerInventory: boolean; + retainedUnknownResourceIds: Set; + actions: ScaleSetReconcileActions; + errors: ScaleSetReconcileError[]; +} + +export class NonRetryableScaleSetError extends Error { + constructor(message: string) { + super(message); + this.name = 'NonRetryableScaleSetError'; + } +} + +export function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +export function safeError( + operation: ScaleSetReconcileOperation, + error: unknown, + details: Pick = {}, +): ScaleSetReconcileError { + return { + operation, + code: safeErrorCode(error), + retryable: isRetryableError(error), + ...details, + }; +} + +function safeErrorCode(error: unknown): string { + if (error instanceof NonRetryableScaleSetError) return 'INVALID_CONFIGURATION'; + if (!isRecord(error)) return 'UNEXPECTED_ERROR'; + for (const candidate of [error.name, error.code]) { + if (typeof candidate === 'string' && /^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(candidate)) { + return candidate; + } + } + return 'UNEXPECTED_ERROR'; +} + +function isRetryableError(error: unknown): boolean { + if (error instanceof NonRetryableScaleSetError) return false; + if (!isRecord(error)) return true; + + const identity = [error.name, error.code] + .filter((candidate): candidate is string => typeof candidate === 'string') + .join(' ') + .toLowerCase(); + if (/accessdenied|unauthor|forbidden|permission|validation|invalid|malformed|unsupported/.test(identity)) { + return false; + } + if (/throttl|timeout|temporar|serviceunavailable|internalserver|network|econn|socket|slowdown/.test(identity)) { + return true; + } + + const metadata = isRecord(error.$metadata) ? error.$metadata : undefined; + const status = [error.status, error.statusCode, metadata?.httpStatusCode].find( + (candidate): candidate is number => typeof candidate === 'number', + ); + if (status !== undefined) { + return status >= 500 || [408, 409, 425, 429].includes(status); + } + return true; +} + +export function throwIfAborted(signal: AbortSignal, error?: unknown): void { + if (signal.aborted || (isRecord(error) && error.name === 'AbortError')) { + signal.throwIfAborted(); + throw error; + } +} + +function resultStatus( + errors: readonly ScaleSetReconcileError[], + current: number, + desired: number, + needsRunnerInventory: boolean, +) { + if (errors.some((error) => !error.retryable)) return 'non_retryable_error' as const; + if (errors.length > 0 || current < desired) return 'retryable_error' as const; + if (needsRunnerInventory) return 'retained' as const; + if (current > desired) return 'retained' as const; + return 'converged' as const; +} + +export function finish(state: MutableReconcileState, desiredRunners: number): ScaleSetReconcileResult { + if (desiredRunners >= 0 && state.currentRunners < desiredRunners && state.errors.length === 0) { + state.errors.push({ + operation: 'reconcile', + code: 'CAPACITY_NOT_PROVISIONED', + retryable: true, + }); + } + return { + status: resultStatus(state.errors, state.currentRunners, desiredRunners, state.needsRunnerInventory), + desiredRunners, + currentRunners: state.currentRunners, + needsRunnerInventory: state.needsRunnerInventory, + actions: state.actions, + errors: state.errors, + }; +} + +export function emptyState(currentRunners: number): MutableReconcileState { + return { + currentRunners, + needsRunnerInventory: false, + retainedUnknownResourceIds: new Set(), + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }; +} + +export function retainUnknown(state: MutableReconcileState, resourceId?: string): void { + if (resourceId === undefined) { + state.actions.retainedUnknown++; + return; + } + if (state.retainedUnknownResourceIds.has(resourceId)) return; + state.retainedUnknownResourceIds.add(resourceId); + state.actions.retainedUnknown++; +} + +export function validateDesiredRunners(desiredRunners: number): ScaleSetReconcileError | undefined { + if (!Number.isSafeInteger(desiredRunners) || desiredRunners < 0 || desiredRunners > 10000) { + return { + operation: 'validate', + code: 'INVALID_DESIRED_RUNNER_COUNT', + retryable: false, + }; + } + return undefined; +} + +export function validateBootTimeout(bootTimeoutMinutes: number): ScaleSetReconcileError | undefined { + if ( + !Number.isSafeInteger(bootTimeoutMinutes) || + bootTimeoutMinutes < 1 || + bootTimeoutMinutes > MAX_BOOT_TIMEOUT_MINUTES + ) { + return { + operation: 'validate', + code: 'INVALID_BOOT_TIMEOUT', + retryable: false, + }; + } + return undefined; +} + +export function validateInventorySignal(runnerInventoryComplete: unknown): ScaleSetReconcileError | undefined { + if (typeof runnerInventoryComplete !== 'boolean') { + return { + operation: 'validate', + code: 'INVALID_RUNNER_INVENTORY_SIGNAL', + retryable: false, + }; + } + return undefined; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts new file mode 100644 index 0000000000..29f6247770 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts @@ -0,0 +1,101 @@ +import type { ScaleSetReconcileRequest, ScaleSetRunnerState } from '../../../../scale-set'; +import type { Ec2RunnerOperations } from '../runners'; +import type { CreateEc2ScaleSetProviderInput } from './configuration'; +import { + indexRunnerStates, + isBusyState, + isSafeScaleDownState, + matchingRunnerState, + type OwnedEc2Runner, +} from './inventory'; +import { retainUnknown, safeError, throwIfAborted, type MutableReconcileState } from './reconcile'; + +async function terminateKnownIdleRunner( + runner: OwnedEc2Runner, + githubState: ScaleSetRunnerState, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runnerOperations: Ec2RunnerOperations, +): Promise { + let removalResult; + try { + removalResult = await request.removeRunner({ + runnerId: githubState.runnerId, + runnerName: githubState.runnerName, + scaleSetId: githubState.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('remove_runner', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } + + if (removalResult.status === 'retained_busy') { + state.actions.retainedBusy++; + return false; + } + if (removalResult.status !== 'removed') { + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; + return false; + } + + try { + await runnerOperations.terminate(runner.instanceId); + state.currentRunners--; + state.actions.terminated++; + return true; + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push( + safeError('terminate', error, { runnerName: githubState.runnerName, resourceId: runner.instanceId }), + ); + retainUnknown(state, runner.instanceId); + return false; + } +} + +export async function scaleDown( + input: CreateEc2ScaleSetProviderInput, + runners: readonly OwnedEc2Runner[], + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runnerOperations: Ec2RunnerOperations, +): Promise { + const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); + const candidates: { runner: OwnedEc2Runner; githubState: ScaleSetRunnerState }[] = []; + + for (const runner of runners) { + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + if (!githubState) { + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; + } else if (isBusyState(githubState)) { + state.actions.retainedBusy++; + } else if (isSafeScaleDownState(githubState)) { + candidates.push({ runner, githubState }); + } else { + retainUnknown(state, runner.instanceId); + if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; + } + } + + candidates.sort((left, right) => { + const launchOrder = (right.runner.launchTime?.getTime() ?? 0) - (left.runner.launchTime?.getTime() ?? 0); + return launchOrder || left.runner.instanceId.localeCompare(right.runner.instanceId); + }); + + let remaining = count; + for (const candidate of candidates) { + if (remaining === 0) break; + request.signal.throwIfAborted(); + if (await terminateKnownIdleRunner(candidate.runner, candidate.githubState, request, state, runnerOperations)) { + remaining--; + } + } +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts new file mode 100644 index 0000000000..1669fc4ba6 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts @@ -0,0 +1,255 @@ +import { DeleteParameterCommand, PutParameterCommand, type SSMClient, type Tag as SsmTag } from '@aws-sdk/client-ssm'; + +import type { GenerateScaleSetJitConfigurationResult, ScaleSetReconcileRequest } from '../../../../scale-set'; +import type { Ec2RunnerOperations } from '../runners'; +import type { CreateEc2ScaleSetProviderInput, Ec2ScaleSetProviderConfig } from './configuration'; +import { + EC2_GITHUB_RUNNER_ID_TAG, + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, + GITHUB_RUNNER_NAME_MAX_LENGTH, + githubScopeHash, + ownershipTags, + runnerIdentityFromGitHubScope, + SCALE_SET_RUNNER_SOURCE, +} from './inventory'; +import { + NonRetryableScaleSetError, + retainUnknown, + safeError, + throwIfAborted, + type MutableReconcileState, +} from './reconcile'; + +const SSM_STANDARD_TIER_THRESHOLD = 4000; +const SSM_ADVANCED_TIER_MAX_BYTES = 8192; + +function jitParameterName(config: Ec2ScaleSetProviderConfig, instanceId: string): string { + return `${config.jitConfigParameterPath}/${instanceId}`; +} + +function jitParameterTags(input: CreateEc2ScaleSetProviderInput, instanceId: string): SsmTag[] { + const reserved = new Set(['InstanceId', EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG, EC2_GITHUB_SCOPE_HASH_TAG]); + return [ + ...(input.configuration.ssmParameterTags ?? []).filter((tag) => tag.Key && !reserved.has(tag.Key)), + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, + ]; +} + +async function publishJitConfiguration( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + encodedJitConfiguration: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + const valueSize = Buffer.byteLength(encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new NonRetryableScaleSetError('JIT configuration must be between 1 and 8192 bytes'); + } + + await ssmClient.send( + new PutParameterCommand({ + Name: jitParameterName(input.configuration, instanceId), + Value: encodedJitConfiguration, + Type: 'SecureString', + KeyId: input.configuration.ssmKmsKeyId, + Overwrite: false, + Tier: valueSize >= SSM_STANDARD_TIER_THRESHOLD ? 'Advanced' : 'Standard', + Tags: jitParameterTags(input, instanceId), + }), + { abortSignal: signal }, + ); +} + +async function bestEffortCancelJitPublication( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + ssmClient: SSMClient, + signal: AbortSignal, +): Promise { + try { + await ssmClient.send(new DeleteParameterCommand({ Name: jitParameterName(input.configuration, instanceId) }), { + abortSignal: signal, + }); + } catch (error) { + throwIfAborted(signal, error); + } +} + +function validateJitResult( + result: GenerateScaleSetJitConfigurationResult, + expectedRunnerName: string, + scaleSetId: number, +): void { + if ( + !Number.isSafeInteger(result.runnerId) || + result.runnerId <= 0 || + result.runnerName !== expectedRunnerName || + result.scaleSetId !== scaleSetId + ) { + throw new NonRetryableScaleSetError('JIT configuration returned an unexpected runner identity'); + } + const valueSize = Buffer.byteLength(result.encodedJitConfiguration, 'utf8'); + if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { + throw new NonRetryableScaleSetError('JIT configuration has an invalid size'); + } +} + +async function terminateUnpublishedRunner( + instanceId: string, + state: MutableReconcileState, + runners: Ec2RunnerOperations, + signal: AbortSignal, +): Promise { + try { + await runners.terminate(instanceId); + state.currentRunners--; + state.actions.terminated++; + } catch (error) { + throwIfAborted(signal, error); + retainUnknown(state, instanceId); + state.errors.push(safeError('terminate', error, { resourceId: instanceId })); + } +} + +async function cleanGitHubRunner( + jit: GenerateScaleSetJitConfigurationResult, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, +): Promise { + try { + await request.removeRunner({ + runnerId: jit.runnerId, + runnerName: jit.runnerName, + scaleSetId: jit.scaleSetId, + signal: request.signal, + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('remove_runner', error, { runnerName: jit.runnerName })); + } +} + +async function configureLaunchedRunner( + input: CreateEc2ScaleSetProviderInput, + instanceId: string, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runners: Ec2RunnerOperations, + ssmClient: SSMClient, +): Promise { + const runnerName = `${input.configuration.runnerNamePrefix}${instanceId}`; + if (runnerName.length > GITHUB_RUNNER_NAME_MAX_LENGTH) { + state.errors.push({ + operation: 'generate_jit_configuration', + code: 'RUNNER_NAME_TOO_LONG', + retryable: false, + resourceId: instanceId, + }); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + let jit: GenerateScaleSetJitConfigurationResult; + try { + jit = await request.generateJitConfiguration({ runnerName, signal: request.signal }); + validateJitResult(jit, runnerName, input.scaleSetId); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('generate_jit_configuration', error, { runnerName, resourceId: instanceId })); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + try { + await runners.tag(instanceId, [ + { Key: EC2_RUNNER_NAME_TAG, Value: jit.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(jit.runnerId) }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, + ]); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + await cleanGitHubRunner(jit, request, state); + await terminateUnpublishedRunner(instanceId, state, runners, request.signal); + return; + } + + try { + await publishJitConfiguration(input, instanceId, jit.encodedJitConfiguration, ssmClient, request.signal); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('publish_jit_configuration', error, { runnerName, resourceId: instanceId })); + await bestEffortCancelJitPublication(input, instanceId, ssmClient, request.signal); + // Main's bootstrap reads before deleting. Even a successful controller-side + // DeleteParameter can race after that read and cannot prove non-consumption. + // Preserve both GitHub and compute state until an exact lifecycle signal is observed. + retainUnknown(state, instanceId); + return; + } + + state.actions.launched++; + try { + await runners.tag(instanceId, [{ Key: EC2_SCALE_SET_STATE_TAG, Value: 'config-published' }]); + } catch (error) { + throwIfAborted(request.signal, error); + // Publication may already have been consumed. Preserve the instance and exact GitHub identity. + retainUnknown(state, instanceId); + state.errors.push(safeError('launch', error, { runnerName, resourceId: instanceId })); + } +} + +export async function scaleUp( + input: CreateEc2ScaleSetProviderInput, + count: number, + request: ScaleSetReconcileRequest, + state: MutableReconcileState, + runners: Ec2RunnerOperations, + ssmClient: SSMClient, +): Promise { + const runnerIdentity = runnerIdentityFromGitHubScope(input.githubScope); + let createResult; + try { + createResult = await runners.create({ + environment: input.configuration.environment, + runnerOwner: runnerIdentity.runnerOwner, + runnerType: runnerIdentity.runnerType, + subnets: input.configuration.subnets, + launchTemplateName: input.configuration.launchTemplateName, + ec2instanceCriteria: input.configuration.ec2instanceCriteria, + ec2OverrideConfig: input.configuration.ec2OverrideConfig, + numberOfRunners: count, + source: SCALE_SET_RUNNER_SOURCE, + amiIdSsmParameterName: input.configuration.amiIdSsmParameterName, + tracingEnabled: input.configuration.tracingEnabled, + onDemandFailoverOnError: input.configuration.onDemandFailoverOnError, + scaleErrors: input.configuration.scaleErrors, + useDedicatedHost: input.configuration.useDedicatedHost, + orchestrationTags: ownershipTags(input), + }); + } catch (error) { + throwIfAborted(request.signal, error); + state.errors.push(safeError('launch', error)); + return; + } + + state.currentRunners += createResult.instances.length; + if (createResult.retryableErrorCount > 0) { + state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_RETRYABLE', retryable: true }); + } + if (createResult.nonRetryableErrorCount > 0) { + state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_NON_RETRYABLE', retryable: false }); + } + + for (const instanceId of createResult.instances) { + request.signal.throwIfAborted(); + await configureLaunchedRunner(input, instanceId, request, state, runners, ssmClient); + } +} From b0f1d4176777b0ab34ee5af902c8107514a58096 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 26 Aug 2026 14:51:24 +0200 Subject: [PATCH 03/19] fix(scale-set): avoid unsafe URL regular expressions --- .../src/client.test.ts | 16 +++++++++++- .../github-actions-scale-set/src/client.ts | 10 +++++--- .../src/config.test.ts | 25 +++++++++++++++++++ .../github-actions-scale-set/src/config.ts | 6 +++-- .../libs/github-actions-scale-set/src/url.ts | 24 ++++++++++++++++++ 5 files changed, 74 insertions(+), 7 deletions(-) create mode 100644 lambdas/libs/github-actions-scale-set/src/url.ts diff --git a/lambdas/libs/github-actions-scale-set/src/client.test.ts b/lambdas/libs/github-actions-scale-set/src/client.test.ts index 3a14a7591d..a1b23c8aef 100644 --- a/lambdas/libs/github-actions-scale-set/src/client.test.ts +++ b/lambdas/libs/github-actions-scale-set/src/client.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'; -import { GitHubActionsScaleSetClient } from './client'; +import { actionsServiceUrl, GitHubActionsScaleSetClient } from './client'; import { SCALE_SET_ERROR_CODES, ScaleSetHttpError, ScaleSetProtocolError } from './errors'; import { ScaleSetFetch } from './types'; @@ -66,6 +66,20 @@ function clientFixture(serviceHandler: ServiceHandler, options: { adminToken?: ( }; } +describe('actionsServiceUrl', () => { + it('normalizes a long trailing slash sequence before joining the request path', () => { + const base = `https://actions.example/tenant/123${'/'.repeat(10_000)}`; + + expect(actionsServiceUrl(base, '').pathname).toBe('/tenant/123'); + expect(actionsServiceUrl(base, '_apis/runtime/runnerscalesets').pathname).toBe( + '/tenant/123/_apis/runtime/runnerscalesets', + ); + const url = actionsServiceUrl(base, '/_apis/runtime/runnerscalesets'); + expect(url.pathname).toBe('/tenant/123/_apis/runtime/runnerscalesets'); + expect(url.searchParams.get('api-version')).toBe('6.0-preview'); + }); +}); + describe('GitHubActionsScaleSetClient', () => { beforeEach(() => { vi.restoreAllMocks(); diff --git a/lambdas/libs/github-actions-scale-set/src/client.ts b/lambdas/libs/github-actions-scale-set/src/client.ts index ec6b98c805..89b4d96cb2 100644 --- a/lambdas/libs/github-actions-scale-set/src/client.ts +++ b/lambdas/libs/github-actions-scale-set/src/client.ts @@ -17,6 +17,7 @@ import { ScaleSetRequestOptions, SystemInfo, } from './types'; +import { trimTrailingSlashes } from './url'; const ADMIN_TOKEN_REFRESH_SKEW_MS = 60_000; const SUCCESS_STATUSES = Array.from({ length: 100 }, (_, index) => index + 200); @@ -67,16 +68,17 @@ interface ActionsRequestOptions extends ScaleSetRequestOptions { } function joinUrlPath(base: string, path: string): string { - if (base === '') { + const normalizedBase = trimTrailingSlashes(base); + if (normalizedBase === '') { if (path === '') { return ''; } return path.startsWith('/') ? path : `/${path}`; } if (path === '') { - return base.replace(/\/+$/, ''); + return normalizedBase; } - return `${base.replace(/\/+$/, '')}${path.startsWith('/') ? '' : '/'}${path}`; + return `${normalizedBase}${path.startsWith('/') ? '' : '/'}${path}`; } export function actionsServiceUrl( @@ -661,6 +663,6 @@ export class GitHubActionsScaleSetClient { 'Actions service admin connection url must be HTTPS and contain no credentials, query, or fragment', ); } - return { url: actionsServiceUrl.toString().replace(/\/$/, ''), token: response.token }; + return { url: trimTrailingSlashes(actionsServiceUrl.toString()), token: response.token }; } } diff --git a/lambdas/libs/github-actions-scale-set/src/config.test.ts b/lambdas/libs/github-actions-scale-set/src/config.test.ts index 35fc866066..316fbab015 100644 --- a/lambdas/libs/github-actions-scale-set/src/config.test.ts +++ b/lambdas/libs/github-actions-scale-set/src/config.test.ts @@ -15,4 +15,29 @@ describe('GitHub configuration URL parsing', () => { isHosted: true, }); }); + + it('normalizes long slash sequences', () => { + const slashSequence = '/'.repeat(10_000); + + expect(parseGitHubConfigUrl(`https://github.com${slashSequence}example${slashSequence}`)).toMatchObject({ + configUrl: new URL('https://github.com/example'), + scope: 'organization', + organization: 'example', + isHosted: true, + }); + }); + + it.each(['https://github.com////', 'https://github.com/org//repository', 'https://github.com/org/repository/extra'])( + 'continues to reject an invalid path after slash normalization: %s', + (configUrl) => { + expect(() => parseGitHubConfigUrl(configUrl)).toThrow(InvalidGitHubConfigUrlError); + }, + ); + + it.each([ + ['https://github.com/org/repository////', 'repository'], + ['https://github.com/enterprises/example////', 'enterprise'], + ])('retains the scope when normalizing %s', (configUrl, scope) => { + expect(parseGitHubConfigUrl(configUrl)).toMatchObject({ scope }); + }); }); diff --git a/lambdas/libs/github-actions-scale-set/src/config.ts b/lambdas/libs/github-actions-scale-set/src/config.ts index 9007167dec..e9b5e1a4a3 100644 --- a/lambdas/libs/github-actions-scale-set/src/config.ts +++ b/lambdas/libs/github-actions-scale-set/src/config.ts @@ -1,3 +1,5 @@ +import { trimSurroundingSlashes, trimTrailingSlashes } from './url'; + export const GITHUB_SCOPES = { enterprise: 'enterprise', organization: 'organization', @@ -44,7 +46,7 @@ function isHostedGitHubUrl(configUrl: URL, forceGhes?: boolean): boolean { export function parseGitHubConfigUrl(configUrl: string, forceGhes?: boolean): ParsedGitHubConfig { let parsedUrl: URL; try { - parsedUrl = new URL(configUrl.trim().replace(/\/+$/, '')); + parsedUrl = new URL(trimTrailingSlashes(configUrl.trim())); } catch (error) { throw new InvalidGitHubConfigUrlError(configUrl, { cause: error }); } @@ -53,7 +55,7 @@ export function parseGitHubConfigUrl(configUrl: string, forceGhes?: boolean): Pa throw new InvalidGitHubConfigUrlError(configUrl); } - const pathParts = parsedUrl.pathname.replace(/^\/+|\/+$/g, '').split('/'); + const pathParts = trimSurroundingSlashes(parsedUrl.pathname).split('/'); const isHosted = isHostedGitHubUrl(parsedUrl, forceGhes); if (pathParts.length === 1 && pathParts[0] !== '') { diff --git a/lambdas/libs/github-actions-scale-set/src/url.ts b/lambdas/libs/github-actions-scale-set/src/url.ts new file mode 100644 index 0000000000..55d9ecb468 --- /dev/null +++ b/lambdas/libs/github-actions-scale-set/src/url.ts @@ -0,0 +1,24 @@ +const FORWARD_SLASH = '/'.charCodeAt(0); + +export function trimTrailingSlashes(value: string): string { + let end = value.length; + while (end > 0 && value.charCodeAt(end - 1) === FORWARD_SLASH) { + end -= 1; + } + + return end === value.length ? value : value.slice(0, end); +} + +export function trimSurroundingSlashes(value: string): string { + let start = 0; + while (start < value.length && value.charCodeAt(start) === FORWARD_SLASH) { + start += 1; + } + + let end = value.length; + while (end > start && value.charCodeAt(end - 1) === FORWARD_SLASH) { + end -= 1; + } + + return start === 0 && end === value.length ? value : value.slice(start, end); +} From 924ab11ed7c8be6458a03156fcbfcd3ba79aecd1 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 26 Aug 2026 15:17:21 +0200 Subject: [PATCH 04/19] test(scale-set): split provider coverage by context --- .../ec2/src/scale-set/configuration.test.ts | 39 + .../aws/ec2/src/scale-set/inventory.test.ts | 142 ++++ .../aws/ec2/src/scale-set/provider.test.ts | 699 +----------------- .../aws/ec2/src/scale-set/reconcile.test.ts | 32 + .../aws/ec2/src/scale-set/scale-down.test.ts | 214 ++++++ .../aws/ec2/src/scale-set/scale-up.test.ts | 152 ++++ .../aws/ec2/src/scale-set/test/fixtures.ts | 112 +++ .../src/scale-set/test/provider-harness.ts | 40 + .../libs/compute-providers/vitest.config.ts | 2 +- 9 files changed, 746 insertions(+), 686 deletions(-) create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts create mode 100644 lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts new file mode 100644 index 0000000000..00feaef2f8 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from 'vitest'; + +import { parseEc2ScaleSetProviderConfig } from './configuration'; +import { EC2_SCALE_SET_ID_TAG } from './inventory'; +import { config } from './test/fixtures'; + +describe('EC2 scale-set provider configuration', () => { + it('strictly parses the supported provider-owned configuration', () => { + expect(parseEc2ScaleSetProviderConfig(config)).toMatchObject(config); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: '' })).toMatchObject({ + runnerNamePrefix: '', + }); + expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: 'r'.repeat(45) })).toMatchObject({ + runnerNamePrefix: 'r'.repeat(45), + }); + }); + + it.each([ + [{ ...config, region: '$(credential)' }], + [{ ...config, subnets: ['subnet-12345678', 'subnet-12345678'] }], + [{ ...config, ec2instanceCriteria: { ...config.ec2instanceCriteria, instanceAllocationStrategy: 'diversified' } }], + [{ ...config, ec2OverrideConfig: { UserData: 'untrusted' } }], + [{ ...config, ssmParameterTags: [{ Key: 'aws:owner', Value: 'untrusted' }] }], + [{ ...config, runnerNamePrefix: 'r'.repeat(46) }], + [{ ...config, bootTimeoutMinutes: 10 }], + ])('rejects invalid or unsupported values instead of forwarding them to AWS', (invalid) => { + expect(() => parseEc2ScaleSetProviderConfig(invalid)).toThrow(); + }); + + it('does not expose configurable EC2 ownership or lifecycle tags', () => { + expect(Object.keys(config)).not.toContain('orchestrationTags'); + expect(() => + parseEc2ScaleSetProviderConfig({ + ...config, + orchestrationTags: [{ Key: EC2_SCALE_SET_ID_TAG, Value: 'another-scale-set' }], + }), + ).toThrow("Unsupported EC2 scale-set configuration field 'configuration.orchestrationTags'"); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts new file mode 100644 index 0000000000..fc03dc3612 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts @@ -0,0 +1,142 @@ +import { createHash } from 'node:crypto'; + +import { CreateFleetCommand, DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { EC2_GITHUB_SCOPE_HASH_TAG, EC2_RUNNER_CONFIG_TAG, EC2_SCALE_SET_ID_TAG } from './inventory'; +import { createRequest, githubScopeHash, githubState, ownedInstance } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set inventory', () => { + it('lists only the exact runner-config and scale-set ownership boundary', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ + ownedInstance('i-owned', { runnerId: 101, runnerName: 'runner-i-owned' }), + ownedInstance('i-other', undefined, { runnerConfigName: 'other' }), + ownedInstance( + 'i-other-scope', + { runnerId: 102, runnerName: 'runner-i-other-scope' }, + { + githubScopeHash: createHash('sha256').update('https://github.com/another', 'utf8').digest('hex'), + }, + ), + ], + }, + ], + }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ status: 'converged', desiredRunners: 1, currentRunners: 1 }); + expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { + Filters: expect.arrayContaining([ + { Name: 'tag:ghr:Application', Values: ['github-action-runner'] }, + { Name: 'tag:ghr:created_by', Values: ['scale-set-service'] }, + { Name: 'tag:ghr:environment', Values: ['unit-test'] }, + { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: ['linux'] }, + { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: ['42'] }, + { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash] }, + ]), + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts a young handed-off instance as serving during its bounded boot window', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-booting', { runnerId: 101, runnerName: 'runner-i-booting' })], + }, + ], + }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:09:59Z').getTime() }).reconcile( + createRequest(), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('uses the orchestration request boot window instead of provider configuration', async () => { + ec2Mock.on(DescribeInstancesCommand).resolves({ + Reservations: [ + { + Instances: [ownedInstance('i-at-timeout', { runnerId: 101, runnerName: 'runner-i-at-timeout' })], + }, + ], + }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:05:00Z').getTime() }).reconcile( + createRequest({ bootTimeoutMinutes: 5 }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { launched: 0, retainedUnknown: 1 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('requests a complete inventory for an old handoff, then counts only its exact online identity', async () => { + const instance = ownedInstance('i-old', { runnerId: 101, runnerName: 'runner-i-old' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const computeProvider = createTestProvider({ now: () => new Date('2026-08-24T10:10:00Z').getTime() }); + + const firstPass = await computeProvider.reconcile(createRequest()); + + expect(firstPass).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { launched: 0, retainedUnknown: 1 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + + const secondPass = await computeProvider.reconcile( + createRequest({ + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-i-old', { status: 'online', lifecycle: 'unknown' })], + }), + ); + + expect(secondPass).toMatchObject({ + status: 'converged', + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 0, retainedUnknown: 0 }, + }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + + it('counts an exact JobStarted identity as serving without waiting for public inventory', async () => { + const instance = ownedInstance('i-started', { runnerId: 101, runnerName: 'runner-i-started' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + + const result = await createTestProvider({ now: () => new Date('2026-08-24T12:00:00Z').getTime() }).reconcile( + createRequest({ + runnerStates: [ + githubState(101, 'runner-i-started', { status: 'unknown', busy: undefined, lifecycle: 'started' }), + ], + }), + ); + + expect(result).toMatchObject({ + status: 'converged', + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 0, retainedUnknown: 0 }, + }); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts index 6fe5f27bcc..67570584ac 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts @@ -1,327 +1,18 @@ -import { createHash } from 'node:crypto'; +import { CreateFleetCommand, DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { PutParameterCommand } from '@aws-sdk/client-ssm'; +import { beforeEach, describe, expect, it } from 'vitest'; -import { - CreateFleetCommand, - CreateTagsCommand, - DescribeInstancesCommand, - EC2Client, - TerminateInstancesCommand, - type Instance, -} from '@aws-sdk/client-ec2'; -import { DeleteParameterCommand, GetParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; -import { mockClient } from 'aws-sdk-client-mock'; -import 'aws-sdk-client-mock-jest/vitest'; -import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { config, createRequest, githubState, ownedInstance } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks, ssmMock } from './test/provider-harness'; -import type { - GenerateScaleSetJitConfigurationResult, - ScaleSetReconcileRequest, - ScaleSetRunnerState, -} from '../../../../scale-set'; -import { - createEc2ScaleSetProvider, - EC2_GITHUB_SCOPE_HASH_TAG, - EC2_GITHUB_RUNNER_ID_TAG, - EC2_RUNNER_CONFIG_TAG, - EC2_RUNNER_NAME_TAG, - EC2_SCALE_SET_ID_TAG, - EC2_SCALE_SET_STATE_TAG, - parseEc2ScaleSetProviderConfig, - type Ec2ScaleSetProviderConfig, -} from './provider'; - -const ec2Mock = mockClient(EC2Client); -const ssmMock = mockClient(SSMClient); -const ec2Client = new EC2Client({ region: 'eu-west-1' }); -const ssmClient = new SSMClient({ region: 'eu-west-1' }); -const signal = new AbortController().signal; -const githubScope = 'https://github.com/example'; -const githubScopeHash = createHash('sha256').update(githubScope, 'utf8').digest('hex'); - -const config: Ec2ScaleSetProviderConfig = { - region: 'eu-west-1', - environment: 'unit-test', - runnerNamePrefix: 'runner-', - jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', - subnets: ['subnet-12345678'], - launchTemplateName: 'unit-test-runners', - ec2instanceCriteria: { - instanceTypes: ['m7i.large'], - targetCapacityType: 'on-demand', - instanceAllocationStrategy: 'lowest-price', - }, - scaleErrors: ['InsufficientInstanceCapacity'], - ssmParameterTags: [{ Key: 'Project', Value: 'runner-tests' }], -}; - -function ownedInstance( - instanceId: string, - identity?: { runnerId: number; runnerName: string }, - overrides: { - runnerConfigName?: string; - scaleSetId?: number; - scaleSetState?: string; - githubScopeHash?: string; - launchTime?: Date; - } = {}, -): Instance { - return { - InstanceId: instanceId, - LaunchTime: overrides.launchTime ?? new Date('2026-08-24T10:00:00Z'), - Tags: [ - { Key: 'ghr:Application', Value: 'github-action-runner' }, - { Key: 'ghr:created_by', Value: 'scale-set-service' }, - { Key: 'ghr:environment', Value: 'unit-test' }, - { Key: EC2_RUNNER_CONFIG_TAG, Value: overrides.runnerConfigName ?? 'linux' }, - { Key: EC2_SCALE_SET_ID_TAG, Value: String(overrides.scaleSetId ?? 42) }, - { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: overrides.githubScopeHash ?? githubScopeHash }, - { - Key: EC2_SCALE_SET_STATE_TAG, - Value: overrides.scaleSetState ?? (identity ? 'config-published' : 'provisioning'), - }, - ...(identity - ? [ - { Key: EC2_RUNNER_NAME_TAG, Value: identity.runnerName }, - { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(identity.runnerId) }, - ] - : []), - ], - }; -} - -function githubState( - runnerId: number, - runnerName: string, - overrides: Partial = {}, -): ScaleSetRunnerState { - return { - runnerId, - runnerName, - scaleSetId: 42, - status: 'online', - busy: false, - lifecycle: 'unknown', - ...overrides, - }; -} - -function jitResult(instanceId = 'i-1234567890abcdef0'): GenerateScaleSetJitConfigurationResult { - return { - encodedJitConfiguration: 'sensitive-encoded-jit-configuration', - runnerId: 101, - runnerName: `runner-${instanceId}`, - scaleSetId: 42, - }; -} - -function createRequest(overrides: Partial = {}): ScaleSetReconcileRequest { - return { - desiredRunners: 1, - bootTimeoutMinutes: 10, - runnerInventoryComplete: false, - runnerStates: [], - signal, - generateJitConfiguration: vi.fn().mockResolvedValue(jitResult()), - removeRunner: vi.fn().mockResolvedValue({ status: 'removed' }), - ...overrides, - }; -} - -function provider( - options: { githubScope?: string; now?: () => number; configuration?: Ec2ScaleSetProviderConfig } = {}, -) { - return createEc2ScaleSetProvider( - { - runnerConfigName: 'linux', - scaleSetId: 42, - githubScope: options.githubScope ?? githubScope, - configuration: options.configuration ?? config, - }, - { - ec2Client, - ssmClient, - now: options.now ?? (() => new Date('2026-08-24T10:05:00Z').getTime()), - }, - ); -} - -beforeEach(() => { - ec2Mock.reset(); - ssmMock.reset(); - ec2Mock.on(CreateTagsCommand).resolves({}); - ec2Mock.on(TerminateInstancesCommand).resolves({}); - ssmMock.on(PutParameterCommand).resolves({}); - ssmMock.on(DeleteParameterCommand).resolves({}); -}); - -describe('EC2 scale-set provider configuration', () => { - it('strictly parses the supported provider-owned configuration', () => { - expect(parseEc2ScaleSetProviderConfig(config)).toMatchObject(config); - expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: '' })).toMatchObject({ - runnerNamePrefix: '', - }); - expect(parseEc2ScaleSetProviderConfig({ ...config, runnerNamePrefix: 'r'.repeat(45) })).toMatchObject({ - runnerNamePrefix: 'r'.repeat(45), - }); - }); - - it.each([ - [{ ...config, region: '$(credential)' }], - [{ ...config, subnets: ['subnet-12345678', 'subnet-12345678'] }], - [{ ...config, ec2instanceCriteria: { ...config.ec2instanceCriteria, instanceAllocationStrategy: 'diversified' } }], - [{ ...config, ec2OverrideConfig: { UserData: 'untrusted' } }], - [{ ...config, ssmParameterTags: [{ Key: 'aws:owner', Value: 'untrusted' }] }], - [{ ...config, runnerNamePrefix: 'r'.repeat(46) }], - [{ ...config, bootTimeoutMinutes: 10 }], - ])('rejects invalid or unsupported values instead of forwarding them to AWS', (invalid) => { - expect(() => parseEc2ScaleSetProviderConfig(invalid)).toThrow(); - }); - - it('does not expose configurable EC2 ownership or lifecycle tags', () => { - expect(Object.keys(config)).not.toContain('orchestrationTags'); - expect(() => - parseEc2ScaleSetProviderConfig({ - ...config, - orchestrationTags: [{ Key: EC2_SCALE_SET_ID_TAG, Value: 'another-scale-set' }], - }), - ).toThrow("Unsupported EC2 scale-set configuration field 'configuration.orchestrationTags'"); - }); +beforeEach(resetAwsMocks); +describe('EC2 scale-set provider orchestration', () => { it('rejects non-canonical GitHub ownership scopes before creating clients', () => { - expect(() => provider({ githubScope: 'https://GITHUB.com/example/' })).toThrow( + expect(() => createTestProvider({ githubScope: 'https://GITHUB.com/example/' })).toThrow( 'githubScope must be a canonical HTTPS GitHub configuration URL', ); }); -}); - -describe('EC2 scale-set reconciliation', () => { - it('lists only the exact runner-config and scale-set ownership boundary', async () => { - ec2Mock.on(DescribeInstancesCommand).resolves({ - Reservations: [ - { - Instances: [ - ownedInstance('i-owned', { runnerId: 101, runnerName: 'runner-i-owned' }), - ownedInstance('i-other', undefined, { runnerConfigName: 'other' }), - ownedInstance( - 'i-other-scope', - { runnerId: 102, runnerName: 'runner-i-other-scope' }, - { - githubScopeHash: createHash('sha256').update('https://github.com/another', 'utf8').digest('hex'), - }, - ), - ], - }, - ], - }); - - const result = await provider().reconcile(createRequest()); - - expect(result).toMatchObject({ status: 'converged', desiredRunners: 1, currentRunners: 1 }); - expect(ec2Mock).toHaveReceivedCommandWith(DescribeInstancesCommand, { - Filters: expect.arrayContaining([ - { Name: 'tag:ghr:Application', Values: ['github-action-runner'] }, - { Name: 'tag:ghr:created_by', Values: ['scale-set-service'] }, - { Name: 'tag:ghr:environment', Values: ['unit-test'] }, - { Name: `tag:${EC2_RUNNER_CONFIG_TAG}`, Values: ['linux'] }, - { Name: `tag:${EC2_SCALE_SET_ID_TAG}`, Values: ['42'] }, - { Name: `tag:${EC2_GITHUB_SCOPE_HASH_TAG}`, Values: [githubScopeHash] }, - ]), - }); - expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); - }); - - it('counts a young handed-off instance as serving during its bounded boot window', async () => { - ec2Mock.on(DescribeInstancesCommand).resolves({ - Reservations: [ - { - Instances: [ownedInstance('i-booting', { runnerId: 101, runnerName: 'runner-i-booting' })], - }, - ], - }); - - const result = await provider({ now: () => new Date('2026-08-24T10:09:59Z').getTime() }).reconcile(createRequest()); - - expect(result).toMatchObject({ - status: 'converged', - currentRunners: 1, - needsRunnerInventory: false, - actions: { launched: 0, retainedUnknown: 0 }, - }); - expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); - }); - - it('uses the orchestration request boot window instead of provider configuration', async () => { - ec2Mock.on(DescribeInstancesCommand).resolves({ - Reservations: [ - { - Instances: [ownedInstance('i-at-timeout', { runnerId: 101, runnerName: 'runner-i-at-timeout' })], - }, - ], - }); - - const result = await provider({ now: () => new Date('2026-08-24T10:05:00Z').getTime() }).reconcile( - createRequest({ bootTimeoutMinutes: 5 }), - ); - - expect(result).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: true, - actions: { launched: 0, retainedUnknown: 1 }, - }); - expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); - }); - - it('requests a complete inventory for an old handoff, then counts only its exact online identity', async () => { - const instance = ownedInstance('i-old', { runnerId: 101, runnerName: 'runner-i-old' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const computeProvider = provider({ now: () => new Date('2026-08-24T10:10:00Z').getTime() }); - - const firstPass = await computeProvider.reconcile(createRequest()); - - expect(firstPass).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: true, - actions: { launched: 0, retainedUnknown: 1 }, - }); - expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); - - const secondPass = await computeProvider.reconcile( - createRequest({ - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-i-old', { status: 'online', lifecycle: 'unknown' })], - }), - ); - - expect(secondPass).toMatchObject({ - status: 'converged', - currentRunners: 1, - needsRunnerInventory: false, - actions: { launched: 0, retainedUnknown: 0 }, - }); - expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); - }); - - it('counts an exact JobStarted identity as serving without waiting for public inventory', async () => { - const instance = ownedInstance('i-started', { runnerId: 101, runnerName: 'runner-i-started' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - - const result = await provider({ now: () => new Date('2026-08-24T12:00:00Z').getTime() }).reconcile( - createRequest({ - runnerStates: [ - githubState(101, 'runner-i-started', { status: 'unknown', busy: undefined, lifecycle: 'started' }), - ], - }), - ); - - expect(result).toMatchObject({ - status: 'converged', - currentRunners: 1, - needsRunnerInventory: false, - actions: { launched: 0, retainedUnknown: 0 }, - }); - }); it('retains an old offline handoff and bounds replacement to one physical surge instance', async () => { const old = ownedInstance('i-old-offline', { runnerId: 100, runnerName: 'runner-i-old-offline' }); @@ -338,7 +29,7 @@ describe('EC2 scale-set reconciliation', () => { .resolvesOnce({ Reservations: [{ Instances: [old] }] }) .resolves({ Reservations: [{ Instances: [old, replacement] }] }); ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacementId] }] }); - const computeProvider = provider({ now: () => new Date('2026-08-24T10:11:00Z').getTime() }); + const computeProvider = createTestProvider({ now: () => new Date('2026-08-24T10:11:00Z').getTime() }); const completeInventory = createRequest({ runnerInventoryComplete: true, runnerStates: [ @@ -388,7 +79,7 @@ describe('EC2 scale-set reconciliation', () => { ], }); ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacement] }] }); - const computeProvider = provider(); + const computeProvider = createTestProvider(); const result = await computeProvider.reconcile(createRequest()); @@ -421,7 +112,7 @@ describe('EC2 scale-set reconciliation', () => { ]; ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: ambiguous }] }); - const result = await provider().reconcile(createRequest()); + const result = await createTestProvider().reconcile(createRequest()); expect(result).toMatchObject({ status: 'retained', @@ -432,375 +123,13 @@ describe('EC2 scale-set reconciliation', () => { expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); }); - it('launches owned compute, verifies JIT identity, and publishes only a SecureString', async () => { - const instanceId = 'i-1234567890abcdef0'; - ec2Mock.on(DescribeInstancesCommand).resolves({}); - ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); - const generateJitConfiguration = vi.fn().mockResolvedValue(jitResult(instanceId)); - - const result = await provider().reconcile(createRequest({ generateJitConfiguration })); - - expect(result).toEqual({ - status: 'converged', - desiredRunners: 1, - currentRunners: 1, - needsRunnerInventory: false, - actions: { launched: 1, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, - errors: [], - }); - expect(generateJitConfiguration).toHaveBeenCalledWith({ runnerName: `runner-${instanceId}`, signal }); - expect(ec2Mock).toHaveReceivedCommandWith(CreateFleetCommand, { - TagSpecifications: expect.arrayContaining([ - expect.objectContaining({ - ResourceType: 'instance', - Tags: expect.arrayContaining([ - { Key: 'ghr:created_by', Value: 'scale-set-service' }, - { Key: 'ghr:environment', Value: 'unit-test' }, - { Key: 'ghr:Owner', Value: 'example' }, - { Key: 'ghr:Type', Value: 'Org' }, - { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, - { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, - { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, - { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, - ]), - }), - ]), - }); - expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { - Name: `${config.jitConfigParameterPath}/${instanceId}`, - Value: 'sensitive-encoded-jit-configuration', - Type: 'SecureString', - Overwrite: false, - Tags: expect.arrayContaining([ - { Key: 'InstanceId', Value: instanceId }, - { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, - { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, - { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, - ]), - }); - }); - - it('resolves an AMI parameter through the provider-owned SSM client', async () => { - const instanceId = 'i-1234567890abcdef0'; - const amiIdSsmParameterName = '/github-action-runners/unit-test/ami'; - ec2Mock.on(DescribeInstancesCommand).resolves({}); - ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); - ssmMock.on(GetParameterCommand).resolves({ Parameter: { Value: 'ami-0123456789abcdef0' } }); - - const result = await provider({ - configuration: { ...config, amiIdSsmParameterName }, - }).reconcile(createRequest()); - - expect(result.actions.launched).toBe(1); - expect(ssmMock).toHaveReceivedCommandWith(GetParameterCommand, { - Name: amiIdSsmParameterName, - WithDecryption: true, - }); - }); - - it('does not remove an unrelated GitHub runner when JIT identity validation fails', async () => { - const instanceId = 'i-1234567890abcdef0'; - ec2Mock.on(DescribeInstancesCommand).resolves({}); - ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); - const removeRunner = vi.fn(); - - const result = await provider().reconcile( - createRequest({ - generateJitConfiguration: vi.fn().mockResolvedValue({ - ...jitResult(instanceId), - runnerName: 'runner-owned-by-another-config', - }), - removeRunner, - }), - ); - - expect(result).toMatchObject({ - status: 'non_retryable_error', - currentRunners: 0, - actions: { launched: 0, terminated: 1 }, - errors: [expect.objectContaining({ operation: 'generate_jit_configuration', retryable: false })], - }); - expect(removeRunner).not.toHaveBeenCalled(); - expect(ssmMock).not.toHaveReceivedCommand(PutParameterCommand); - expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: [instanceId] }); - }); - - it('retains compute when failed JIT publication cannot be safely cancelled', async () => { - const instanceId = 'i-1234567890abcdef0'; - ec2Mock.on(DescribeInstancesCommand).resolves({}); - ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); - ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('redacted secret'), { name: 'TimeoutError' })); - ssmMock.on(DeleteParameterCommand).rejects(Object.assign(new Error('missing'), { name: 'ParameterNotFound' })); - const removeRunner = vi.fn(); - - const result = await provider().reconcile(createRequest({ removeRunner })); - - expect(result).toMatchObject({ - status: 'retryable_error', - currentRunners: 1, - actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, - errors: [expect.objectContaining({ operation: 'publish_jit_configuration', code: 'TimeoutError' })], - }); - expect(JSON.stringify(result)).not.toContain('redacted secret'); - expect(removeRunner).not.toHaveBeenCalled(); - expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); - }); - - it('does not treat a successful DeleteParameter as proof that bootstrap did not read JIT first', async () => { - const instanceId = 'i-1234567890abcdef0'; - ec2Mock.on(DescribeInstancesCommand).resolves({}); - ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); - ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('throttled'), { name: 'ThrottlingException' })); - ssmMock.on(DeleteParameterCommand).resolves({}); - const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); - - const result = await provider().reconcile(createRequest({ removeRunner })); - - expect(result).toMatchObject({ - status: 'retryable_error', - currentRunners: 1, - actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, - }); - expect(ssmMock).toHaveReceivedCommandWith(DeleteParameterCommand, { - Name: `${config.jitConfigParameterPath}/${instanceId}`, - }); - expect(removeRunner).not.toHaveBeenCalled(); - expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); - }); - - it('terminates only exact known-idle or completed runners and retains busy or unknown runners', async () => { - const completed = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); - const busy = ownedInstance('i-busy', { runnerId: 102, runnerName: 'runner-busy' }); - const unknown = ownedInstance('i-unknown', { runnerId: 103, runnerName: 'runner-unknown' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [completed, busy, unknown] }] }); - const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); - - const result = await provider().reconcile( - createRequest({ - desiredRunners: 2, - runnerInventoryComplete: true, - runnerStates: [ - githubState(101, 'runner-completed', { status: 'offline', busy: undefined, lifecycle: 'completed' }), - githubState(102, 'runner-busy', { busy: true, lifecycle: 'started' }), - ], - removeRunner, - }), - ); - - expect(result).toEqual({ - status: 'converged', - desiredRunners: 2, - currentRunners: 2, - needsRunnerInventory: false, - actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 1 }, - errors: [], - }); - expect(removeRunner).toHaveBeenCalledTimes(1); - expect(removeRunner).toHaveBeenCalledWith({ - runnerId: 101, - runnerName: 'runner-completed', - scaleSetId: 42, - signal, - }); - expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-completed'] }); - expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-busy'] }); - expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-unknown'] }); - }); - - it('uses a typed inventory signal for a conservative first pass and exact second pass', async () => { - const instance = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); - const computeProvider = provider(); - - const firstPass = await computeProvider.reconcile( - createRequest({ desiredRunners: 0, runnerStates: [], removeRunner }), - ); - - expect(firstPass).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: true, - actions: { terminated: 0, retainedUnknown: 1 }, - errors: [], - }); - expect(removeRunner).not.toHaveBeenCalled(); - - const secondPass = await computeProvider.reconcile( - createRequest({ - desiredRunners: 0, - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-completed', { lifecycle: 'completed', status: 'offline' })], - removeRunner, - }), - ); - - expect(secondPass).toMatchObject({ - status: 'converged', - currentRunners: 0, - needsRunnerInventory: false, - actions: { terminated: 1 }, - }); - expect(removeRunner).toHaveBeenCalledTimes(1); - }); - - it('never lets a completed lifecycle marker override a current busy signal', async () => { - const instance = ownedInstance('i-completed-busy', { runnerId: 101, runnerName: 'runner-completed-busy' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const removeRunner = vi.fn(); - - const result = await provider().reconcile( - createRequest({ - desiredRunners: 0, - runnerInventoryComplete: true, - runnerStates: [ - githubState(101, 'runner-completed-busy', { - lifecycle: 'completed', - status: 'online', - busy: true, - }), - ], - removeRunner, - }), - ); - - expect(result).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: false, - actions: { terminated: 0, retainedBusy: 1 }, - errors: [], - }); - expect(removeRunner).not.toHaveBeenCalled(); - expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); - }); - - it('retains a runner without an error when the exact removal check observes that it became busy', async () => { - const instance = ownedInstance('i-raced-busy', { runnerId: 101, runnerName: 'runner-raced-busy' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_busy' }); - - const result = await provider().reconcile( - createRequest({ - desiredRunners: 0, - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-raced-busy')], - removeRunner, - }), - ); - - expect(result).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: false, - actions: { terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, - errors: [], - }); - expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); - }); - - it('retains a runner and requests inventory when exact removal observes identity drift', async () => { - const instance = ownedInstance('i-raced-unknown', { runnerId: 101, runnerName: 'runner-raced-unknown' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_unknown' }); - - const result = await provider().reconcile( - createRequest({ - desiredRunners: 0, - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-raced-unknown')], - removeRunner, - }), - ); - - expect(result).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: false, - actions: { terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, - errors: [], - }); - expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); - }); - - it('does not trust a mutable EC2 GitHub-runner-id tag when controller identity disagrees', async () => { - const instance = ownedInstance('i-mismatch', { runnerId: 999, runnerName: 'runner-exact' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const removeRunner = vi.fn(); - - const result = await provider().reconcile( - createRequest({ - desiredRunners: 0, - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-exact')], - removeRunner, - }), - ); - - expect(result).toMatchObject({ - status: 'retained', - currentRunners: 1, - actions: { terminated: 0, retainedUnknown: 1 }, - errors: [], - }); - expect(removeRunner).not.toHaveBeenCalled(); - expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); - }); - - it('does not terminate compute when exact GitHub removal fails', async () => { - const instance = ownedInstance('i-idle', { runnerId: 101, runnerName: 'runner-idle' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const removeRunner = vi - .fn() - .mockRejectedValue(Object.assign(new Error('must not leak'), { name: 'ServiceUnavailable' })); - - const result = await provider().reconcile( - createRequest({ - desiredRunners: 0, - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-idle')], - removeRunner, - }), - ); - - expect(result).toMatchObject({ - status: 'retryable_error', - currentRunners: 1, - actions: { terminated: 0, retainedUnknown: 1 }, - errors: expect.arrayContaining([expect.objectContaining({ operation: 'remove_runner', retryable: true })]), - }); - expect(JSON.stringify(result)).not.toContain('must not leak'); - expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); - }); - - it('rejects an invalid desired count without touching AWS', async () => { - const result = await provider().reconcile(createRequest({ desiredRunners: -1 })); - - expect(result).toMatchObject({ - status: 'non_retryable_error', - desiredRunners: -1, - currentRunners: 0, - errors: [{ operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT', retryable: false }], - }); - expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); - }); - - it.each([0, 121, 1.5])('rejects invalid orchestration boot timeout %s without touching AWS', async (value) => { - const result = await provider().reconcile(createRequest({ bootTimeoutMinutes: value })); - - expect(result).toMatchObject({ - status: 'non_retryable_error', - currentRunners: 0, - errors: [{ operation: 'validate', code: 'INVALID_BOOT_TIMEOUT', retryable: false }], - }); - expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); - }); - it('propagates cancellation instead of converting shutdown into a retry result', async () => { const abort = new AbortController(); abort.abort(new Error('service stopping')); - await expect(provider().reconcile(createRequest({ signal: abort.signal }))).rejects.toThrow('service stopping'); + await expect(createTestProvider().reconcile(createRequest({ signal: abort.signal }))).rejects.toThrow( + 'service stopping', + ); expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); }); }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts new file mode 100644 index 0000000000..a9ffd959f5 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts @@ -0,0 +1,32 @@ +import { DescribeInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it } from 'vitest'; + +import { createRequest } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set reconciliation validation', () => { + it('rejects an invalid desired count without touching AWS', async () => { + const result = await createTestProvider().reconcile(createRequest({ desiredRunners: -1 })); + + expect(result).toMatchObject({ + status: 'non_retryable_error', + desiredRunners: -1, + currentRunners: 0, + errors: [{ operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT', retryable: false }], + }); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); + + it.each([0, 121, 1.5])('rejects invalid orchestration boot timeout %s without touching AWS', async (value) => { + const result = await createTestProvider().reconcile(createRequest({ bootTimeoutMinutes: value })); + + expect(result).toMatchObject({ + status: 'non_retryable_error', + currentRunners: 0, + errors: [{ operation: 'validate', code: 'INVALID_BOOT_TIMEOUT', retryable: false }], + }); + expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts new file mode 100644 index 0000000000..da0e447fa2 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts @@ -0,0 +1,214 @@ +import { DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { createRequest, githubState, ownedInstance, signal } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set scale down', () => { + it('terminates only exact known-idle or completed runners and retains busy or unknown runners', async () => { + const completed = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + const busy = ownedInstance('i-busy', { runnerId: 102, runnerName: 'runner-busy' }); + const unknown = ownedInstance('i-unknown', { runnerId: 103, runnerName: 'runner-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [completed, busy, unknown] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 2, + runnerInventoryComplete: true, + runnerStates: [ + githubState(101, 'runner-completed', { status: 'offline', busy: undefined, lifecycle: 'completed' }), + githubState(102, 'runner-busy', { busy: true, lifecycle: 'started' }), + ], + removeRunner, + }), + ); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 2, + currentRunners: 2, + needsRunnerInventory: false, + actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + expect(removeRunner).toHaveBeenCalledWith({ + runnerId: 101, + runnerName: 'runner-completed', + scaleSetId: 42, + signal, + }); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-completed'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-busy'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-unknown'] }); + }); + + it('uses a typed inventory signal for a conservative first pass and exact second pass', async () => { + const instance = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + const computeProvider = createTestProvider(); + + const firstPass = await computeProvider.reconcile( + createRequest({ desiredRunners: 0, runnerStates: [], removeRunner }), + ); + + expect(firstPass).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + + const secondPass = await computeProvider.reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-completed', { lifecycle: 'completed', status: 'offline' })], + removeRunner, + }), + ); + + expect(secondPass).toMatchObject({ + status: 'converged', + currentRunners: 0, + needsRunnerInventory: false, + actions: { terminated: 1 }, + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + }); + + it('never lets a completed lifecycle marker override a current busy signal', async () => { + const instance = ownedInstance('i-completed-busy', { runnerId: 101, runnerName: 'runner-completed-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [ + githubState(101, 'runner-completed-busy', { + lifecycle: 'completed', + status: 'online', + busy: true, + }), + ], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: false, + actions: { terminated: 0, retainedBusy: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner without an error when the exact removal check observes that it became busy', async () => { + const instance = ownedInstance('i-raced-busy', { runnerId: 101, runnerName: 'runner-raced-busy' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_busy' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-raced-busy')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: false, + actions: { terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('retains a runner and requests inventory when exact removal observes identity drift', async () => { + const instance = ownedInstance('i-raced-unknown', { runnerId: 101, runnerName: 'runner-raced-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn().mockResolvedValue({ status: 'retained_unknown' }); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-raced-unknown')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: false, + actions: { terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not trust a mutable EC2 GitHub-runner-id tag when controller identity disagrees', async () => { + const instance = ownedInstance('i-mismatch', { runnerId: 999, runnerName: 'runner-exact' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-exact')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not terminate compute when exact GitHub removal fails', async () => { + const instance = ownedInstance('i-idle', { runnerId: 101, runnerName: 'runner-idle' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); + const removeRunner = vi + .fn() + .mockRejectedValue(Object.assign(new Error('must not leak'), { name: 'ServiceUnavailable' })); + + const result = await createTestProvider().reconcile( + createRequest({ + desiredRunners: 0, + runnerInventoryComplete: true, + runnerStates: [githubState(101, 'runner-idle')], + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'retryable_error', + currentRunners: 1, + actions: { terminated: 0, retainedUnknown: 1 }, + errors: expect.arrayContaining([expect.objectContaining({ operation: 'remove_runner', retryable: true })]), + }); + expect(JSON.stringify(result)).not.toContain('must not leak'); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts new file mode 100644 index 0000000000..a92b08476e --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts @@ -0,0 +1,152 @@ +import { CreateFleetCommand, DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { DeleteParameterCommand, GetParameterCommand, PutParameterCommand } from '@aws-sdk/client-ssm'; +import { beforeEach, describe, expect, it, vi } from 'vitest'; + +import { + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from './inventory'; +import { config, createRequest, githubScopeHash, jitResult, signal } from './test/fixtures'; +import { createTestProvider, ec2Mock, resetAwsMocks, ssmMock } from './test/provider-harness'; + +beforeEach(resetAwsMocks); + +describe('EC2 scale-set scale up', () => { + it('launches owned compute, verifies JIT identity, and publishes only a SecureString', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const generateJitConfiguration = vi.fn().mockResolvedValue(jitResult(instanceId)); + + const result = await createTestProvider().reconcile(createRequest({ generateJitConfiguration })); + + expect(result).toEqual({ + status: 'converged', + desiredRunners: 1, + currentRunners: 1, + needsRunnerInventory: false, + actions: { launched: 1, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, + errors: [], + }); + expect(generateJitConfiguration).toHaveBeenCalledWith({ runnerName: `runner-${instanceId}`, signal }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateFleetCommand, { + TagSpecifications: expect.arrayContaining([ + expect.objectContaining({ + ResourceType: 'instance', + Tags: expect.arrayContaining([ + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: 'ghr:Owner', Value: 'example' }, + { Key: 'ghr:Type', Value: 'Org' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, + ]), + }), + ]), + }); + expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + Value: 'sensitive-encoded-jit-configuration', + Type: 'SecureString', + Overwrite: false, + Tags: expect.arrayContaining([ + { Key: 'InstanceId', Value: instanceId }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + ]), + }); + }); + + it('resolves an AMI parameter through the provider-owned SSM client', async () => { + const instanceId = 'i-1234567890abcdef0'; + const amiIdSsmParameterName = '/github-action-runners/unit-test/ami'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(GetParameterCommand).resolves({ Parameter: { Value: 'ami-0123456789abcdef0' } }); + + const result = await createTestProvider({ + configuration: { ...config, amiIdSsmParameterName }, + }).reconcile(createRequest()); + + expect(result.actions.launched).toBe(1); + expect(ssmMock).toHaveReceivedCommandWith(GetParameterCommand, { + Name: amiIdSsmParameterName, + WithDecryption: true, + }); + }); + + it('does not remove an unrelated GitHub runner when JIT identity validation fails', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile( + createRequest({ + generateJitConfiguration: vi.fn().mockResolvedValue({ + ...jitResult(instanceId), + runnerName: 'runner-owned-by-another-config', + }), + removeRunner, + }), + ); + + expect(result).toMatchObject({ + status: 'non_retryable_error', + currentRunners: 0, + actions: { launched: 0, terminated: 1 }, + errors: [expect.objectContaining({ operation: 'generate_jit_configuration', retryable: false })], + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ssmMock).not.toHaveReceivedCommand(PutParameterCommand); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: [instanceId] }); + }); + + it('retains compute when failed JIT publication cannot be safely cancelled', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('redacted secret'), { name: 'TimeoutError' })); + ssmMock.on(DeleteParameterCommand).rejects(Object.assign(new Error('missing'), { name: 'ParameterNotFound' })); + const removeRunner = vi.fn(); + + const result = await createTestProvider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'retryable_error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + errors: [expect.objectContaining({ operation: 'publish_jit_configuration', code: 'TimeoutError' })], + }); + expect(JSON.stringify(result)).not.toContain('redacted secret'); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); + + it('does not treat a successful DeleteParameter as proof that bootstrap did not read JIT first', async () => { + const instanceId = 'i-1234567890abcdef0'; + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [instanceId] }] }); + ssmMock.on(PutParameterCommand).rejects(Object.assign(new Error('throttled'), { name: 'ThrottlingException' })); + ssmMock.on(DeleteParameterCommand).resolves({}); + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); + + const result = await createTestProvider().reconcile(createRequest({ removeRunner })); + + expect(result).toMatchObject({ + status: 'retryable_error', + currentRunners: 1, + actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, + }); + expect(ssmMock).toHaveReceivedCommandWith(DeleteParameterCommand, { + Name: `${config.jitConfigParameterPath}/${instanceId}`, + }); + expect(removeRunner).not.toHaveBeenCalled(); + expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); + }); +}); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts new file mode 100644 index 0000000000..abed373830 --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts @@ -0,0 +1,112 @@ +import { createHash } from 'node:crypto'; + +import type { Instance } from '@aws-sdk/client-ec2'; +import { vi } from 'vitest'; + +import type { + GenerateScaleSetJitConfigurationResult, + ScaleSetReconcileRequest, + ScaleSetRunnerState, +} from '../../../../../scale-set'; +import type { Ec2ScaleSetProviderConfig } from '../configuration'; +import { + EC2_GITHUB_SCOPE_HASH_TAG, + EC2_GITHUB_RUNNER_ID_TAG, + EC2_RUNNER_CONFIG_TAG, + EC2_RUNNER_NAME_TAG, + EC2_SCALE_SET_ID_TAG, + EC2_SCALE_SET_STATE_TAG, +} from '../inventory'; + +export const signal = new AbortController().signal; +export const githubScope = 'https://github.com/example'; +export const githubScopeHash = createHash('sha256').update(githubScope, 'utf8').digest('hex'); + +export const config: Ec2ScaleSetProviderConfig = { + region: 'eu-west-1', + environment: 'unit-test', + runnerNamePrefix: 'runner-', + jitConfigParameterPath: '/github-action-runners/unit-test/runners/tokens', + subnets: ['subnet-12345678'], + launchTemplateName: 'unit-test-runners', + ec2instanceCriteria: { + instanceTypes: ['m7i.large'], + targetCapacityType: 'on-demand', + instanceAllocationStrategy: 'lowest-price', + }, + scaleErrors: ['InsufficientInstanceCapacity'], + ssmParameterTags: [{ Key: 'Project', Value: 'runner-tests' }], +}; + +export function ownedInstance( + instanceId: string, + identity?: { runnerId: number; runnerName: string }, + overrides: { + runnerConfigName?: string; + scaleSetId?: number; + scaleSetState?: string; + githubScopeHash?: string; + launchTime?: Date; + } = {}, +): Instance { + return { + InstanceId: instanceId, + LaunchTime: overrides.launchTime ?? new Date('2026-08-24T10:00:00Z'), + Tags: [ + { Key: 'ghr:Application', Value: 'github-action-runner' }, + { Key: 'ghr:created_by', Value: 'scale-set-service' }, + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: overrides.runnerConfigName ?? 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: String(overrides.scaleSetId ?? 42) }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: overrides.githubScopeHash ?? githubScopeHash }, + { + Key: EC2_SCALE_SET_STATE_TAG, + Value: overrides.scaleSetState ?? (identity ? 'config-published' : 'provisioning'), + }, + ...(identity + ? [ + { Key: EC2_RUNNER_NAME_TAG, Value: identity.runnerName }, + { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(identity.runnerId) }, + ] + : []), + ], + }; +} + +export function githubState( + runnerId: number, + runnerName: string, + overrides: Partial = {}, +): ScaleSetRunnerState { + return { + runnerId, + runnerName, + scaleSetId: 42, + status: 'online', + busy: false, + lifecycle: 'unknown', + ...overrides, + }; +} + +export function jitResult(instanceId = 'i-1234567890abcdef0'): GenerateScaleSetJitConfigurationResult { + return { + encodedJitConfiguration: 'sensitive-encoded-jit-configuration', + runnerId: 101, + runnerName: `runner-${instanceId}`, + scaleSetId: 42, + }; +} + +export function createRequest(overrides: Partial = {}): ScaleSetReconcileRequest { + return { + desiredRunners: 1, + bootTimeoutMinutes: 10, + runnerInventoryComplete: false, + runnerStates: [], + signal, + generateJitConfiguration: vi.fn().mockResolvedValue(jitResult()), + removeRunner: vi.fn().mockResolvedValue({ status: 'removed' }), + ...overrides, + }; +} diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts new file mode 100644 index 0000000000..510009001a --- /dev/null +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/provider-harness.ts @@ -0,0 +1,40 @@ +import { CreateTagsCommand, EC2Client, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { DeleteParameterCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { mockClient } from 'aws-sdk-client-mock'; +import 'aws-sdk-client-mock-jest/vitest'; + +import type { Ec2ScaleSetProviderConfig } from '../configuration'; +import { createEc2ScaleSetProvider } from '../provider'; +import { config, githubScope } from './fixtures'; + +export const ec2Mock = mockClient(EC2Client); +export const ssmMock = mockClient(SSMClient); +const ec2Client = new EC2Client({ region: 'eu-west-1' }); +const ssmClient = new SSMClient({ region: 'eu-west-1' }); + +export function createTestProvider( + options: { githubScope?: string; now?: () => number; configuration?: Ec2ScaleSetProviderConfig } = {}, +) { + return createEc2ScaleSetProvider( + { + runnerConfigName: 'linux', + scaleSetId: 42, + githubScope: options.githubScope ?? githubScope, + configuration: options.configuration ?? config, + }, + { + ec2Client, + ssmClient, + now: options.now ?? (() => new Date('2026-08-24T10:05:00Z').getTime()), + }, + ); +} + +export function resetAwsMocks(): void { + ec2Mock.reset(); + ssmMock.reset(); + ec2Mock.on(CreateTagsCommand).resolves({}); + ec2Mock.on(TerminateInstancesCommand).resolves({}); + ssmMock.on(PutParameterCommand).resolves({}); + ssmMock.on(DeleteParameterCommand).resolves({}); +} diff --git a/lambdas/libs/compute-providers/vitest.config.ts b/lambdas/libs/compute-providers/vitest.config.ts index fd62b358c0..24c4554995 100644 --- a/lambdas/libs/compute-providers/vitest.config.ts +++ b/lambdas/libs/compute-providers/vitest.config.ts @@ -16,7 +16,7 @@ export default mergeConfig(defaultConfig, { 'core/**/*.ts', 'aws/**/*.ts', ], - exclude: ['**/*.test.ts', '**/*.d.ts', 'templates/**/*'], + exclude: ['**/*.test.ts', '**/test/**/*.ts', '**/*.d.ts', 'templates/**/*'], thresholds: { statements: 96.16, branches: 95.32, From b8b5101c866d6261a152bf1552025e911bb589dc Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Wed, 26 Aug 2026 15:23:07 +0200 Subject: [PATCH 05/19] test(scale-set): clarify invalid region fixture --- .../aws/ec2/src/scale-set/configuration.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts index 00feaef2f8..ea539eff9b 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts @@ -16,7 +16,7 @@ describe('EC2 scale-set provider configuration', () => { }); it.each([ - [{ ...config, region: '$(credential)' }], + [{ ...config, region: 'eu-west-one' }], [{ ...config, subnets: ['subnet-12345678', 'subnet-12345678'] }], [{ ...config, ec2instanceCriteria: { ...config.ec2instanceCriteria, instanceAllocationStrategy: 'diversified' } }], [{ ...config, ec2OverrideConfig: { UserData: 'untrusted' } }], From fb95b1ab3d031ca36137afbd0bd0a9cf558419a6 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Thu, 27 Aug 2026 21:25:12 +0200 Subject: [PATCH 06/19] refactor(scale-set): remove control-plane retry policy --- .../ec2/src/scale-set/configuration.test.ts | 1 + .../aws/ec2/src/scale-set/configuration.ts | 57 ++++---- .../aws/ec2/src/scale-set/provider.ts | 2 +- .../aws/ec2/src/scale-set/reconcile.test.ts | 8 +- .../aws/ec2/src/scale-set/reconcile.ts | 39 +---- .../aws/ec2/src/scale-set/scale-down.test.ts | 11 +- .../aws/ec2/src/scale-set/scale-down.ts | 6 +- .../aws/ec2/src/scale-set/scale-up.test.ts | 49 ++++++- .../aws/ec2/src/scale-set/scale-up.ts | 25 ++-- .../aws/ec2/src/scale-set/test/fixtures.ts | 1 - .../libs/compute-providers/scale-set.test.ts | 1 - lambdas/libs/compute-providers/scale-set.ts | 3 +- .../libs/github-actions-scale-set/README.md | 6 +- lambdas/services/scale-set/README.md | 3 +- .../services/scale-set/src/reconciler.test.ts | 135 ++++++++++++++++-- lambdas/services/scale-set/src/reconciler.ts | 75 ++++++++-- 16 files changed, 290 insertions(+), 132 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts index ea539eff9b..d2e50b34d7 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts @@ -20,6 +20,7 @@ describe('EC2 scale-set provider configuration', () => { [{ ...config, subnets: ['subnet-12345678', 'subnet-12345678'] }], [{ ...config, ec2instanceCriteria: { ...config.ec2instanceCriteria, instanceAllocationStrategy: 'diversified' } }], [{ ...config, ec2OverrideConfig: { UserData: 'untrusted' } }], + [{ ...config, scaleErrors: ['ThrottlingException'] }], [{ ...config, ssmParameterTags: [{ Key: 'aws:owner', Value: 'untrusted' }] }], [{ ...config, runnerNamePrefix: 'r'.repeat(46) }], [{ ...config, bootTimeoutMinutes: 10 }], diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts index 17fa0cce37..1d03728cb8 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.ts @@ -1,7 +1,7 @@ import type { Tag as SsmTag } from '@aws-sdk/client-ssm'; import type { Ec2OverrideConfig, RunnerInputParameters } from '../runners.d'; -import { isRecord, NonRetryableScaleSetError } from './reconcile'; +import { isRecord, Ec2ScaleSetValidationError } from './reconcile'; const SPOT_ALLOCATION_STRATEGIES = new Set([ 'lowest-price', @@ -24,7 +24,6 @@ export interface Ec2ScaleSetProviderConfig { amiIdSsmParameterName?: string; tracingEnabled?: boolean; onDemandFailoverOnError?: string[]; - scaleErrors: string[]; useDedicatedHost?: boolean; ssmKmsKeyId?: string; ssmParameterTags?: SsmTag[]; @@ -40,20 +39,20 @@ export interface CreateEc2ScaleSetProviderInput { function rejectUnknownKeys(value: Record, allowedKeys: ReadonlySet, name: string): void { const unknownKey = Object.keys(value).find((key) => !allowedKeys.has(key)); if (unknownKey !== undefined) { - throw new NonRetryableScaleSetError(`Unsupported EC2 scale-set configuration field '${name}.${unknownKey}'`); + throw new Ec2ScaleSetValidationError(`Unsupported EC2 scale-set configuration field '${name}.${unknownKey}'`); } } function requireString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { if (typeof value !== 'string' || value.length === 0 || value.length > maximumLength || !pattern.test(value)) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); } return value; } function requirePossiblyEmptyString(value: unknown, name: string, pattern: RegExp, maximumLength: number): string { if (typeof value !== 'string' || value.length > maximumLength || !pattern.test(value)) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); } return value; } @@ -66,7 +65,7 @@ function optionalString(value: unknown, name: string, pattern: RegExp, maximumLe function optionalBoolean(value: unknown, name: string): boolean | undefined { if (value === undefined) return undefined; if (typeof value !== 'boolean') { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); } return value; } @@ -79,11 +78,11 @@ function requireStringArray( allowEmpty = false, ): string[] { if (!Array.isArray(value) || (!allowEmpty && value.length === 0) || value.length > 100) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); } const parsed = value.map((item, index) => requireString(item, `${name}[${index}]`, pattern, maximumItemLength)); if (new Set(parsed).size !== parsed.length) { - throw new NonRetryableScaleSetError(`EC2 scale-set configuration field '${name}' contains duplicate values`); + throw new Ec2ScaleSetValidationError(`EC2 scale-set configuration field '${name}' contains duplicate values`); } return parsed; } @@ -91,14 +90,14 @@ function requireStringArray( function parseInstanceTypePriorities(value: unknown): Record | undefined { if (value === undefined) return undefined; if (!isRecord(value)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'instanceTypePriorities'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'instanceTypePriorities'"); } const result = Object.create(null) as Record; for (const [instanceType, priority] of Object.entries(value)) { requireString(instanceType, 'instanceTypePriorities key', /^[a-z0-9][a-z0-9.-]*$/, 64); if (typeof priority !== 'number' || !Number.isSafeInteger(priority) || priority < 0 || priority > 1000) { - throw new NonRetryableScaleSetError( + throw new Ec2ScaleSetValidationError( `Invalid EC2 scale-set configuration priority for instance type '${instanceType}'`, ); } @@ -109,12 +108,12 @@ function parseInstanceTypePriorities(value: unknown): Record | u function requireSsmTagValue(value: unknown): string { if (typeof value !== 'string' || value.length > 256) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); } for (const character of value) { const codePoint = character.codePointAt(0)!; if (codePoint < 32 || codePoint === 127) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags.Value'"); } } return value; @@ -123,7 +122,7 @@ function requireSsmTagValue(value: unknown): string { function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { if (value === undefined) return undefined; if (!isRecord(value)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2OverrideConfig'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ec2OverrideConfig'"); } const supportedKeys = new Set([ @@ -137,7 +136,7 @@ function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { 'ImageId', ]); if (Object.keys(value).some((key) => !supportedKeys.has(key))) { - throw new NonRetryableScaleSetError('EC2 scale-set configuration contains an unsupported launch override'); + throw new Ec2ScaleSetValidationError('EC2 scale-set configuration contains an unsupported launch override'); } const weightedCapacity = value.WeightedCapacity; @@ -147,7 +146,7 @@ function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { ['Priority', priority], ] as const) { if (number !== undefined && (typeof number !== 'number' || !Number.isFinite(number) || number < 0)) { - throw new NonRetryableScaleSetError(`Invalid EC2 scale-set configuration field '${name}'`); + throw new Ec2ScaleSetValidationError(`Invalid EC2 scale-set configuration field '${name}'`); } } @@ -181,19 +180,19 @@ function parseEc2OverrideConfig(value: unknown): Ec2OverrideConfig | undefined { function parseSsmTags(value: unknown): SsmTag[] | undefined { if (value === undefined) return undefined; if (!Array.isArray(value) || value.length > 45) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); } const tags: SsmTag[] = []; const keys = new Set(); for (const item of value) { if (!isRecord(item)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ssmParameterTags'"); } const key = requireString(item.Key, 'ssmParameterTags.Key', /^[A-Za-z0-9_.:/=+@-]+$/, 128); const tagValue = requireSsmTagValue(item.Value); if (key.toLowerCase().startsWith('aws:') || keys.has(key)) { - throw new NonRetryableScaleSetError(`Invalid or duplicate SSM tag key '${key}'`); + throw new Ec2ScaleSetValidationError(`Invalid or duplicate SSM tag key '${key}'`); } keys.add(key); tags.push({ Key: key, Value: tagValue }); @@ -203,7 +202,7 @@ function parseSsmTags(value: unknown): SsmTag[] | undefined { export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProviderConfig { if (!isRecord(value)) { - throw new NonRetryableScaleSetError('EC2 scale-set provider configuration must be an object'); + throw new Ec2ScaleSetValidationError('EC2 scale-set provider configuration must be an object'); } rejectUnknownKeys( value, @@ -219,7 +218,6 @@ export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProvi 'amiIdSsmParameterName', 'tracingEnabled', 'onDemandFailoverOnError', - 'scaleErrors', 'useDedicatedHost', 'ssmKmsKeyId', 'ssmParameterTags', @@ -227,7 +225,7 @@ export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProvi 'configuration', ); if (!isRecord(value.ec2instanceCriteria)) { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'ec2instanceCriteria'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'ec2instanceCriteria'"); } rejectUnknownKeys( value.ec2instanceCriteria, @@ -243,7 +241,7 @@ export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProvi const targetCapacityType = value.ec2instanceCriteria.targetCapacityType; if (targetCapacityType !== 'on-demand' && targetCapacityType !== 'spot') { - throw new NonRetryableScaleSetError("Invalid EC2 scale-set configuration field 'targetCapacityType'"); + throw new Ec2ScaleSetValidationError("Invalid EC2 scale-set configuration field 'targetCapacityType'"); } const instanceAllocationStrategy = requireString( value.ec2instanceCriteria.instanceAllocationStrategy, @@ -254,7 +252,7 @@ export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProvi const allowedAllocationStrategies = targetCapacityType === 'spot' ? SPOT_ALLOCATION_STRATEGIES : ON_DEMAND_ALLOCATION_STRATEGIES; if (!allowedAllocationStrategies.has(instanceAllocationStrategy)) { - throw new NonRetryableScaleSetError( + throw new Ec2ScaleSetValidationError( `Invalid allocation strategy '${instanceAllocationStrategy}' for '${targetCapacityType}' capacity`, ); } @@ -305,7 +303,6 @@ export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProvi 128, true, ), - scaleErrors: requireStringArray(value.scaleErrors ?? [], 'scaleErrors', /^[A-Za-z0-9._-]+$/, 128, true), useDedicatedHost: optionalBoolean(value.useDedicatedHost, 'useDedicatedHost'), ssmKmsKeyId: optionalString(value.ssmKmsKeyId, 'ssmKmsKeyId', /^[A-Za-z0-9_:/+=,.@-]+$/, 2048), ssmParameterTags: parseSsmTags(value.ssmParameterTags), @@ -315,35 +312,35 @@ export function parseEc2ScaleSetProviderConfig(value: unknown): Ec2ScaleSetProvi export function validateFactoryInput(input: CreateEc2ScaleSetProviderInput): void { requireString(input.runnerConfigName, 'runnerConfigName', /^[A-Za-z0-9][A-Za-z0-9._-]*$/, 128); if (!Number.isSafeInteger(input.scaleSetId) || input.scaleSetId <= 0) { - throw new NonRetryableScaleSetError('scaleSetId must be a positive safe integer'); + throw new Ec2ScaleSetValidationError('scaleSetId must be a positive safe integer'); } validateCanonicalGitHubScope(input.githubScope); } function validateCanonicalGitHubScope(value: unknown): string { if (typeof value !== 'string' || value.length === 0 || value.length > 2048) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); } let url: URL; try { url = new URL(value); } catch { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); } if (url.protocol !== 'https:' || url.username || url.password || url.search || url.hash) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); } const parts = url.pathname .replace(/^\/+|\/+$/g, '') .split('/') .filter(Boolean); if (parts.length < 1 || parts.length > 2 || (parts[0].toLowerCase() === 'enterprises' && parts.length !== 2)) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); } url.pathname = `/${parts.join('/')}`; const canonical = url.toString().replace(/\/$/, ''); if (canonical !== value) { - throw new NonRetryableScaleSetError('githubScope must be a canonical HTTPS GitHub configuration URL'); + throw new Ec2ScaleSetValidationError('githubScope must be a canonical HTTPS GitHub configuration URL'); } return value; } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts index 8bfa056b17..fd0cb54b99 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts @@ -64,7 +64,7 @@ export function createEc2ScaleSetProvider( }; validateFactoryInput(normalizedInput); const clients = createClients(normalizedInput.configuration, dependencies); - const runnerClient = createEc2RunnerClient(clients.ec2Client, clients.ssmClient); + const runnerClient = createEc2RunnerClient(clients.ec2Client); const now = dependencies.now ?? Date.now; return { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts index a9ffd959f5..a462b935b8 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.test.ts @@ -11,11 +11,11 @@ describe('EC2 scale-set reconciliation validation', () => { const result = await createTestProvider().reconcile(createRequest({ desiredRunners: -1 })); expect(result).toMatchObject({ - status: 'non_retryable_error', + status: 'error', desiredRunners: -1, currentRunners: 0, - errors: [{ operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT', retryable: false }], }); + expect(result.errors).toEqual([{ operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT' }]); expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); }); @@ -23,10 +23,10 @@ describe('EC2 scale-set reconciliation validation', () => { const result = await createTestProvider().reconcile(createRequest({ bootTimeoutMinutes: value })); expect(result).toMatchObject({ - status: 'non_retryable_error', + status: 'error', currentRunners: 0, - errors: [{ operation: 'validate', code: 'INVALID_BOOT_TIMEOUT', retryable: false }], }); + expect(result.errors).toEqual([{ operation: 'validate', code: 'INVALID_BOOT_TIMEOUT' }]); expect(ec2Mock).not.toHaveReceivedCommand(DescribeInstancesCommand); }); }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts index 4a94a6e842..d4341abe1e 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts @@ -15,10 +15,10 @@ export interface MutableReconcileState { errors: ScaleSetReconcileError[]; } -export class NonRetryableScaleSetError extends Error { +export class Ec2ScaleSetValidationError extends Error { constructor(message: string) { super(message); - this.name = 'NonRetryableScaleSetError'; + this.name = 'Ec2ScaleSetValidationError'; } } @@ -34,13 +34,12 @@ export function safeError( return { operation, code: safeErrorCode(error), - retryable: isRetryableError(error), ...details, }; } function safeErrorCode(error: unknown): string { - if (error instanceof NonRetryableScaleSetError) return 'INVALID_CONFIGURATION'; + if (error instanceof Ec2ScaleSetValidationError) return 'INVALID_CONFIGURATION'; if (!isRecord(error)) return 'UNEXPECTED_ERROR'; for (const candidate of [error.name, error.code]) { if (typeof candidate === 'string' && /^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(candidate)) { @@ -50,31 +49,6 @@ function safeErrorCode(error: unknown): string { return 'UNEXPECTED_ERROR'; } -function isRetryableError(error: unknown): boolean { - if (error instanceof NonRetryableScaleSetError) return false; - if (!isRecord(error)) return true; - - const identity = [error.name, error.code] - .filter((candidate): candidate is string => typeof candidate === 'string') - .join(' ') - .toLowerCase(); - if (/accessdenied|unauthor|forbidden|permission|validation|invalid|malformed|unsupported/.test(identity)) { - return false; - } - if (/throttl|timeout|temporar|serviceunavailable|internalserver|network|econn|socket|slowdown/.test(identity)) { - return true; - } - - const metadata = isRecord(error.$metadata) ? error.$metadata : undefined; - const status = [error.status, error.statusCode, metadata?.httpStatusCode].find( - (candidate): candidate is number => typeof candidate === 'number', - ); - if (status !== undefined) { - return status >= 500 || [408, 409, 425, 429].includes(status); - } - return true; -} - export function throwIfAborted(signal: AbortSignal, error?: unknown): void { if (signal.aborted || (isRecord(error) && error.name === 'AbortError')) { signal.throwIfAborted(); @@ -88,8 +62,7 @@ function resultStatus( desired: number, needsRunnerInventory: boolean, ) { - if (errors.some((error) => !error.retryable)) return 'non_retryable_error' as const; - if (errors.length > 0 || current < desired) return 'retryable_error' as const; + if (errors.length > 0 || current < desired) return 'error' as const; if (needsRunnerInventory) return 'retained' as const; if (current > desired) return 'retained' as const; return 'converged' as const; @@ -100,7 +73,6 @@ export function finish(state: MutableReconcileState, desiredRunners: number): Sc state.errors.push({ operation: 'reconcile', code: 'CAPACITY_NOT_PROVISIONED', - retryable: true, }); } return { @@ -138,7 +110,6 @@ export function validateDesiredRunners(desiredRunners: number): ScaleSetReconcil return { operation: 'validate', code: 'INVALID_DESIRED_RUNNER_COUNT', - retryable: false, }; } return undefined; @@ -153,7 +124,6 @@ export function validateBootTimeout(bootTimeoutMinutes: number): ScaleSetReconci return { operation: 'validate', code: 'INVALID_BOOT_TIMEOUT', - retryable: false, }; } return undefined; @@ -164,7 +134,6 @@ export function validateInventorySignal(runnerInventoryComplete: unknown): Scale return { operation: 'validate', code: 'INVALID_RUNNER_INVENTORY_SIGNAL', - retryable: false, }; } return undefined; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts index da0e447fa2..1b3b5e0c60 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts @@ -203,11 +203,18 @@ describe('EC2 scale-set scale down', () => { ); expect(result).toMatchObject({ - status: 'retryable_error', + status: 'error', currentRunners: 1, actions: { terminated: 0, retainedUnknown: 1 }, - errors: expect.arrayContaining([expect.objectContaining({ operation: 'remove_runner', retryable: true })]), }); + expect(result.errors).toEqual([ + { + operation: 'remove_runner', + code: 'ServiceUnavailable', + runnerName: 'runner-idle', + resourceId: 'i-idle', + }, + ]); expect(JSON.stringify(result)).not.toContain('must not leak'); expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts index 29f6247770..b94b09f455 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts @@ -1,5 +1,5 @@ import type { ScaleSetReconcileRequest, ScaleSetRunnerState } from '../../../../scale-set'; -import type { Ec2RunnerOperations } from '../runners'; +import type { Ec2RunnerResourceOperations } from '../runners'; import type { CreateEc2ScaleSetProviderInput } from './configuration'; import { indexRunnerStates, @@ -15,7 +15,7 @@ async function terminateKnownIdleRunner( githubState: ScaleSetRunnerState, request: ScaleSetReconcileRequest, state: MutableReconcileState, - runnerOperations: Ec2RunnerOperations, + runnerOperations: Ec2RunnerResourceOperations, ): Promise { let removalResult; try { @@ -65,7 +65,7 @@ export async function scaleDown( count: number, request: ScaleSetReconcileRequest, state: MutableReconcileState, - runnerOperations: Ec2RunnerOperations, + runnerOperations: Ec2RunnerResourceOperations, ): Promise { const runnerStateIndex = indexRunnerStates(request.runnerStates, input.scaleSetId); const candidates: { runner: OwnedEc2Runner; githubState: ScaleSetRunnerState }[] = []; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts index a92b08476e..13a62a0c6f 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts @@ -97,11 +97,18 @@ describe('EC2 scale-set scale up', () => { ); expect(result).toMatchObject({ - status: 'non_retryable_error', + status: 'error', currentRunners: 0, actions: { launched: 0, terminated: 1 }, - errors: [expect.objectContaining({ operation: 'generate_jit_configuration', retryable: false })], }); + expect(result.errors).toEqual([ + { + operation: 'generate_jit_configuration', + code: 'INVALID_CONFIGURATION', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); expect(removeRunner).not.toHaveBeenCalled(); expect(ssmMock).not.toHaveReceivedCommand(PutParameterCommand); expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: [instanceId] }); @@ -118,11 +125,18 @@ describe('EC2 scale-set scale up', () => { const result = await createTestProvider().reconcile(createRequest({ removeRunner })); expect(result).toMatchObject({ - status: 'retryable_error', + status: 'error', currentRunners: 1, actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, - errors: [expect.objectContaining({ operation: 'publish_jit_configuration', code: 'TimeoutError' })], }); + expect(result.errors).toEqual([ + { + operation: 'publish_jit_configuration', + code: 'TimeoutError', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); expect(JSON.stringify(result)).not.toContain('redacted secret'); expect(removeRunner).not.toHaveBeenCalled(); expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); @@ -139,14 +153,39 @@ describe('EC2 scale-set scale up', () => { const result = await createTestProvider().reconcile(createRequest({ removeRunner })); expect(result).toMatchObject({ - status: 'retryable_error', + status: 'error', currentRunners: 1, actions: { launched: 0, terminated: 0, retainedUnknown: 1 }, }); + expect(result.errors).toEqual([ + { + operation: 'publish_jit_configuration', + code: 'ThrottlingException', + runnerName: `runner-${instanceId}`, + resourceId: instanceId, + }, + ]); expect(ssmMock).toHaveReceivedCommandWith(DeleteParameterCommand, { Name: `${config.jitConfigParameterPath}/${instanceId}`, }); expect(removeRunner).not.toHaveBeenCalled(); expect(ec2Mock).not.toHaveReceivedCommand(TerminateInstancesCommand); }); + + it.each(['ThrottlingException', 'InvalidParameterValue'])( + 'collapses EC2 launch failure %s into one scale-set error', + async (errorCode) => { + ec2Mock.on(DescribeInstancesCommand).resolves({}); + ec2Mock.on(CreateFleetCommand).resolves({ Errors: [{ ErrorCode: errorCode }] }); + + const result = await createTestProvider().reconcile(createRequest()); + + expect(result).toMatchObject({ + status: 'error', + currentRunners: 0, + actions: { launched: 0, terminated: 0 }, + }); + expect(result.errors).toEqual([{ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }]); + }, + ); }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts index 1669fc4ba6..41c4f943ad 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts @@ -1,7 +1,7 @@ import { DeleteParameterCommand, PutParameterCommand, type SSMClient, type Tag as SsmTag } from '@aws-sdk/client-ssm'; import type { GenerateScaleSetJitConfigurationResult, ScaleSetReconcileRequest } from '../../../../scale-set'; -import type { Ec2RunnerOperations } from '../runners'; +import type { Ec2RunnerResourceOperations } from '../runners'; import type { CreateEc2ScaleSetProviderInput, Ec2ScaleSetProviderConfig } from './configuration'; import { EC2_GITHUB_RUNNER_ID_TAG, @@ -17,7 +17,7 @@ import { SCALE_SET_RUNNER_SOURCE, } from './inventory'; import { - NonRetryableScaleSetError, + Ec2ScaleSetValidationError, retainUnknown, safeError, throwIfAborted, @@ -51,7 +51,7 @@ async function publishJitConfiguration( ): Promise { const valueSize = Buffer.byteLength(encodedJitConfiguration, 'utf8'); if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { - throw new NonRetryableScaleSetError('JIT configuration must be between 1 and 8192 bytes'); + throw new Ec2ScaleSetValidationError('JIT configuration must be between 1 and 8192 bytes'); } await ssmClient.send( @@ -94,18 +94,18 @@ function validateJitResult( result.runnerName !== expectedRunnerName || result.scaleSetId !== scaleSetId ) { - throw new NonRetryableScaleSetError('JIT configuration returned an unexpected runner identity'); + throw new Ec2ScaleSetValidationError('JIT configuration returned an unexpected runner identity'); } const valueSize = Buffer.byteLength(result.encodedJitConfiguration, 'utf8'); if (valueSize === 0 || valueSize > SSM_ADVANCED_TIER_MAX_BYTES) { - throw new NonRetryableScaleSetError('JIT configuration has an invalid size'); + throw new Ec2ScaleSetValidationError('JIT configuration has an invalid size'); } } async function terminateUnpublishedRunner( instanceId: string, state: MutableReconcileState, - runners: Ec2RunnerOperations, + runners: Ec2RunnerResourceOperations, signal: AbortSignal, ): Promise { try { @@ -142,7 +142,7 @@ async function configureLaunchedRunner( instanceId: string, request: ScaleSetReconcileRequest, state: MutableReconcileState, - runners: Ec2RunnerOperations, + runners: Ec2RunnerResourceOperations, ssmClient: SSMClient, ): Promise { const runnerName = `${input.configuration.runnerNamePrefix}${instanceId}`; @@ -150,7 +150,6 @@ async function configureLaunchedRunner( state.errors.push({ operation: 'generate_jit_configuration', code: 'RUNNER_NAME_TOO_LONG', - retryable: false, resourceId: instanceId, }); await terminateUnpublishedRunner(instanceId, state, runners, request.signal); @@ -211,7 +210,7 @@ export async function scaleUp( count: number, request: ScaleSetReconcileRequest, state: MutableReconcileState, - runners: Ec2RunnerOperations, + runners: Ec2RunnerResourceOperations, ssmClient: SSMClient, ): Promise { const runnerIdentity = runnerIdentityFromGitHubScope(input.githubScope); @@ -230,7 +229,6 @@ export async function scaleUp( amiIdSsmParameterName: input.configuration.amiIdSsmParameterName, tracingEnabled: input.configuration.tracingEnabled, onDemandFailoverOnError: input.configuration.onDemandFailoverOnError, - scaleErrors: input.configuration.scaleErrors, useDedicatedHost: input.configuration.useDedicatedHost, orchestrationTags: ownershipTags(input), }); @@ -241,11 +239,8 @@ export async function scaleUp( } state.currentRunners += createResult.instances.length; - if (createResult.retryableErrorCount > 0) { - state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_RETRYABLE', retryable: true }); - } - if (createResult.nonRetryableErrorCount > 0) { - state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_NON_RETRYABLE', retryable: false }); + if (createResult.failedInstanceCount > 0) { + state.errors.push({ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }); } for (const instanceId of createResult.instances) { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts index abed373830..2f770dfdbf 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts @@ -34,7 +34,6 @@ export const config: Ec2ScaleSetProviderConfig = { targetCapacityType: 'on-demand', instanceAllocationStrategy: 'lowest-price', }, - scaleErrors: ['InsufficientInstanceCapacity'], ssmParameterTags: [{ Key: 'Project', Value: 'runner-tests' }], }; diff --git a/lambdas/libs/compute-providers/scale-set.test.ts b/lambdas/libs/compute-providers/scale-set.test.ts index e3cd187ce6..f56e044d86 100644 --- a/lambdas/libs/compute-providers/scale-set.test.ts +++ b/lambdas/libs/compute-providers/scale-set.test.ts @@ -19,7 +19,6 @@ const configuration = { targetCapacityType: 'on-demand', instanceAllocationStrategy: 'lowest-price', }, - scaleErrors: [], }; describe('scale-set compute-provider registry', () => { diff --git a/lambdas/libs/compute-providers/scale-set.ts b/lambdas/libs/compute-providers/scale-set.ts index 6f06a91c08..2575f95384 100644 --- a/lambdas/libs/compute-providers/scale-set.ts +++ b/lambdas/libs/compute-providers/scale-set.ts @@ -62,7 +62,7 @@ export interface ScaleSetReconcileRequest { removeRunner: RemoveScaleSetRunner; } -export type ScaleSetReconcileStatus = 'converged' | 'retained' | 'retryable_error' | 'non_retryable_error'; +export type ScaleSetReconcileStatus = 'converged' | 'retained' | 'error'; export type ScaleSetReconcileOperation = | 'validate' @@ -78,7 +78,6 @@ export type ScaleSetReconcileOperation = export interface ScaleSetReconcileError { operation: ScaleSetReconcileOperation; code: string; - retryable: boolean; runnerName?: string; resourceId?: string; } diff --git a/lambdas/libs/github-actions-scale-set/README.md b/lambdas/libs/github-actions-scale-set/README.md index 3892c678a2..6696627a11 100644 --- a/lambdas/libs/github-actions-scale-set/README.md +++ b/lambdas/libs/github-actions-scale-set/README.md @@ -49,20 +49,20 @@ const session = await client.createMessageSessionClient(scaleSet.id!, 'listener- try { const message = await session.getMessage(0, 20); if (message) { + await session.deleteMessage(message.messageId); + const availableIds = message.jobAvailableMessages.map((job) => job.runnerRequestId); await session.acquireJobs(availableIds); // Reconcile compute from message.statistics.totalAssignedJobs and use // client.generateJitRunnerConfig(...) for every runner being created. - - await session.deleteMessage(message.messageId); } } finally { await session.close(); } ``` -Treat encoded JIT configurations and all access tokens as secrets. A message should be acknowledged only after its compute and completion handling succeeds so it can be redelivered after a failure. +Treat encoded JIT configurations and all access tokens as secrets. The upstream Go listener acknowledges a message before job acquisition and scaling callbacks; a later callback failure is returned from the listener and does not cause message redelivery. The retry values shown above are the defaults. Automatic transport retries apply only to idempotent methods (`GET`, `HEAD`, `OPTIONS`, `PUT`, and `DELETE`). Non-idempotent `POST` and `PATCH` operations, including JIT generation, session creation, and job acquisition, are attempted once so a lost response cannot cause the operation to be replayed. `Retry-After` is honored for eligible 429/5xx responses and capped by `maxBackoffMs`. Caller cancellation interrupts both an active request and retry backoff. diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md index 7bc3a46e01..38fadad978 100644 --- a/lambdas/services/scale-set/README.md +++ b/lambdas/services/scale-set/README.md @@ -57,7 +57,6 @@ The service reads every direct child under the SSM path with paginated `GetParam "instanceAllocationStrategy": "price-capacity-optimized" }, "onDemandFailoverOnError": [], - "scaleErrors": [], "useDedicatedHost": false, "ssmParameterTags": [] } @@ -88,7 +87,7 @@ Demand is calculated as `max(totalAssignedJobs, min(maxRunners, minRunners + tot The public GitHub runner inventory is not fetched on ordinary steady-state or scale-up polls. A compute provider explicitly requests one bounded, owner-scope inventory refresh when it needs to verify old handed-off capacity or perform safe scale-down; owner inventory is briefly shared across reconcilers. The first provider pass is marked lifecycle-only and the second is explicitly marked inventory-complete, so the provider cannot mistake a post-restart gap for an authoritative absence. Runner deletion executes inside the serialized reconcile loop, re-fetches the Actions identity by name, and then performs a fresh public GitHub lookup to verify the exact ID/name and confirm the runner is not busy before issuing the delete. -Message acknowledgement is last: acquire available jobs, update lifecycle state, complete the idempotent compute reconciliation, then delete the message. A failure leaves the message available for redelivery. A typed busy/unknown retention is processed and acknowledged without closing the session; it is not treated as an API failure. +Messages follow the upstream scale-set listener order: acknowledge first, then acquire available jobs, update lifecycle state, and reconcile compute. Provider failures therefore stop that reconciler after the message has been acknowledged; provider results expose one error outcome rather than the control-plane scaling retry classification. A typed busy/unknown retention remains a successful reconciliation. Session and transport failures are handled separately by bounded client retries or session recreation. The EC2 provider counts a `config-published` instance as serving only during the orchestration request's boot window (`bootTimeoutMinutes`, default `10`) or after an exact online or `JobStarted` identity is observed. After the window, offline or unknown capacity is retained rather than terminated, and the complete inventory pass allows it to stop suppressing a replacement. Instances left in an earlier or unknown publication state are also retained for operator recovery and never terminated speculatively. EC2 ownership includes a SHA-256 hash of the canonical GitHub configuration scope, preventing the same runner-config name and numeric scale-set ID in another GitHub scope from colliding. A bounded one-instance physical surge may replace ambiguous capacity; once that ceiling is reached, the provider reports retained capacity instead of creating an unbounded replacement loop. diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index a83baa87c9..f226431dbf 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -114,7 +114,7 @@ function fixture(options: { } describe('ScaleSetReconciler', () => { - it('acknowledges only after acquisition, lifecycle observation, and successful reconciliation', async () => { + it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { const order: string[] = []; const abort = new AbortController(); const session = { @@ -126,7 +126,6 @@ describe('ScaleSetReconciler', () => { }), deleteMessage: vi.fn(async () => { order.push('delete'); - abort.abort(); }), close: vi.fn(), }; @@ -137,11 +136,13 @@ describe('ScaleSetReconciler', () => { expect(request.runnerStates).toContainEqual( expect.objectContaining({ runnerId: 5, runnerName: 'runner-5', lifecycle: 'started' }), ); + abort.abort(); return result(); }); const { dependencies } = fixture({ session, reconcile }); await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); - expect(order).toEqual(['acquire', 'reconcile', 'delete']); + expect(order).toEqual(['delete', 'acquire', 'reconcile']); + expect(session.deleteMessage).toHaveBeenCalledWith(7, { signal: abort.signal }); expect(dependencies.computeProviders.create).toHaveBeenCalledWith('ec2', { runnerConfigName: 'linux', scaleSetId: 42, @@ -235,7 +236,7 @@ describe('ScaleSetReconciler', () => { ); }); - it('leaves a message unacknowledged when reconciliation fails', async () => { + it('acknowledges and stops when reconciliation rejects', async () => { const abort = new AbortController(); const session = { session: { statistics: undefined }, @@ -244,10 +245,101 @@ describe('ScaleSetReconciler', () => { deleteMessage: vi.fn(), close: vi.fn(), }; - const { dependencies } = fixture({ session, reconcile: vi.fn().mockRejectedValue(new Error('temporary')) }); + const { dependencies } = fixture({ session, reconcile: vi.fn().mockRejectedValue(new Error('provider failed')) }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + expect(status.markReconnecting).not.toHaveBeenCalled(); + expect(dependencies.sleep).not.toHaveBeenCalled(); + }); + + it('does not process a message when acknowledgement fails', async () => { + const abort = new AbortController(); + const reconcile = vi.fn(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn(), + deleteMessage: vi.fn().mockRejectedValue(new Error('acknowledgement failed')), + close: vi.fn(), + }; + const { dependencies } = fixture({ session, reconcile }); dependencies.sleep = vi.fn(async () => abort.abort()); - await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); - expect(session.deleteMessage).not.toHaveBeenCalled(); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(session.deleteMessage).toHaveBeenCalledOnce(); + expect(session.acquireJobs).not.toHaveBeenCalled(); + expect(reconcile).not.toHaveBeenCalled(); + expect(status.markReconnecting).toHaveBeenCalledOnce(); + }); + + it('acknowledges and stops when the provider returns an error result', async () => { + const abort = new AbortController(); + const order: string[] = []; + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(async () => { + order.push('delete'); + }), + close: vi.fn(), + }; + const reconcile = vi.fn(async () => { + order.push('reconcile'); + return result({ + status: 'error', + currentRunners: 0, + errors: [{ operation: 'launch', code: 'ThrottlingException' }], + }); + }); + const { dependencies } = fixture({ session, reconcile }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(order).toEqual(['delete', 'reconcile']); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); + expect(status.markReconnecting).not.toHaveBeenCalled(); + expect(dependencies.sleep).not.toHaveBeenCalled(); + }); + + it('does not request inventory after a provider error result', async () => { + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + acquireJobs: vi.fn().mockResolvedValue([99]), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const reconcile = vi.fn().mockResolvedValue( + result({ + status: 'error', + currentRunners: 0, + needsRunnerInventory: true, + errors: [{ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }], + }), + ); + const { client, dependencies } = fixture({ session, reconcile }); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(new AbortController().signal, status); + + expect(reconcile).toHaveBeenCalledOnce(); + expect(client.listGitHubRunners).not.toHaveBeenCalled(); + expect(client.listRunners).not.toHaveBeenCalled(); + expect(status.markFailed).toHaveBeenCalledWith( + expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), + ); }); it('re-fetches exact state in the serialized loop and acknowledges a typed busy retention', async () => { @@ -262,7 +354,6 @@ describe('ScaleSetReconciler', () => { }), deleteMessage: vi.fn(async () => { order.push('delete'); - abort.abort(); }), close: vi.fn(), }; @@ -271,6 +362,7 @@ describe('ScaleSetReconciler', () => { await expect(request.removeRunner({ runnerId: 5, runnerName: 'runner-5', scaleSetId: 42 })).resolves.toEqual({ status: 'retained_busy', }); + abort.abort(); return result({ status: 'retained', currentRunners: 2, @@ -288,7 +380,7 @@ describe('ScaleSetReconciler', () => { }); await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); - expect(order).toEqual(['acquire', 'reconcile', 'actions-refetch', 'github-refetch', 'delete']); + expect(order).toEqual(['delete', 'acquire', 'reconcile', 'actions-refetch', 'github-refetch']); expect(client.removeRunner).not.toHaveBeenCalled(); expect(session.deleteMessage).toHaveBeenCalledOnce(); expect(session.acquireJobs).toHaveBeenCalledTimes(1); @@ -327,12 +419,24 @@ describe('reconciler helpers', () => { it.each([ { status: 'unexpected' }, + { status: 'retryable_error' }, + { status: 'non_retryable_error' }, + { retryable: true }, { needsRunnerInventory: 'yes' }, { actions: { launched: 0, terminated: 0, retainedBusy: -1, retainedUnknown: 0 } }, - { errors: [{ operation: 'shell', code: 'BAD', retryable: false }] }, - { errors: [{ operation: 'list', code: 'contains spaces', retryable: false }] }, - { errors: [{ operation: 'list', code: 'BAD!CODE', retryable: false }] }, - { errors: [{ operation: 'list', code: 'BAD\nCODE', retryable: false }] }, + { actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0, retryable: true } }, + { status: 'converged', errors: [{ operation: 'list', code: 'UNEXPECTED_ERROR' }] }, + { status: 'retained', errors: [{ operation: 'list', code: 'UNEXPECTED_ERROR' }] }, + { status: 'error', errors: [] }, + { currentRunners: 0 }, + { currentRunners: 2 }, + { needsRunnerInventory: true }, + { status: 'retained' }, + { errors: [{ operation: 'shell', code: 'BAD' }] }, + { errors: [{ operation: 'list', code: 'contains spaces' }] }, + { errors: [{ operation: 'list', code: 'BAD!CODE' }] }, + { errors: [{ operation: 'list', code: 'BAD\nCODE' }] }, + { errors: [{ operation: 'list', code: 'BAD', retryable: true }] }, ])('rejects malformed compute-provider result metadata: %o', (overrides) => { expect(() => validateProviderResult({ ...result(), ...overrides } as ScaleSetReconcileResult, 1)).toThrow( /scale-set compute provider returned (?:an? )?invalid/, @@ -343,9 +447,10 @@ describe('reconciler helpers', () => { expect(() => validateProviderResult( result({ + status: 'error', errors: [ - { operation: 'list', code: 'AccessDeniedException', retryable: false }, - { operation: 'launch', code: 'ThrottlingException', retryable: true }, + { operation: 'list', code: 'AccessDeniedException' }, + { operation: 'launch', code: 'ThrottlingException' }, ], }), 1, diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index f34c931926..0cacd6847f 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -12,6 +12,7 @@ import { import type { ScaleSetComputeProvider, ScaleSetComputeProviderFactoryInput, + ScaleSetReconcileRequest, ScaleSetReconcileResult, ScaleSetRunnerLifecycle, ScaleSetRunnerState, @@ -89,10 +90,15 @@ interface LifecycleObservation { export class ScaleSetProviderReconciliationError extends Error { constructor( - readonly result: ScaleSetReconcileResult, - readonly retryable: boolean, + readonly result?: ScaleSetReconcileResult, + options?: ErrorOptions, ) { - super(`scale-set compute provider returned ${result.status}`); + super( + result === undefined + ? 'scale-set compute provider reconciliation failed' + : `scale-set compute provider returned ${result.status}`, + options, + ); this.name = 'ScaleSetProviderReconciliationError'; } } @@ -171,13 +177,11 @@ export class ScaleSetReconciler { } latestStatistics = message.statistics; lastMessageId = message.messageId; + await session.deleteMessage(message.messageId, { signal }); const requestIds = uniqueRequestIds(message); if (requestIds.length > 0) await session.acquireJobs(requestIds, { signal }); this.observeLifecycle(message); await this.reconcile(client, provider, latestStatistics, signal); - // Acknowledge only after the idempotent provider reconciliation has - // succeeded. Failures intentionally leave the message for redelivery. - await session.deleteMessage(message.messageId, { signal }); this.pruneCompletedLifecycle(message); } madeProgress = true; @@ -299,7 +303,7 @@ export class ScaleSetReconciler { return { status: 'removed' as const }; }, }; - let result = await provider.reconcile({ + let result = await this.reconcileProvider(provider, { desiredRunners, bootTimeoutMinutes: this.config.bootTimeoutMinutes, runnerInventoryComplete: false, @@ -307,9 +311,10 @@ export class ScaleSetReconciler { ...callbacks, }); validateProviderResult(result, desiredRunners); + throwIfProviderError(result); if (result.needsRunnerInventory) { const inventory = await this.loadScaleSetInventory(client, signal); - result = await provider.reconcile({ + result = await this.reconcileProvider(provider, { desiredRunners, bootTimeoutMinutes: this.config.bootTimeoutMinutes, runnerInventoryComplete: true, @@ -317,6 +322,7 @@ export class ScaleSetReconciler { ...callbacks, }); validateProviderResult(result, desiredRunners); + throwIfProviderError(result); if (result.needsRunnerInventory) { throw new ScaleSetProtocolError( 'scale-set compute provider requested inventory after a complete inventory pass', @@ -339,8 +345,17 @@ export class ScaleSetReconciler { }); return; } - if (result.status !== 'converged') { - throw new ScaleSetProviderReconciliationError(result, result.status === 'retryable_error'); + } + + private async reconcileProvider( + provider: ScaleSetComputeProvider, + request: ScaleSetReconcileRequest, + ): Promise { + try { + return await provider.reconcile(request); + } catch (error) { + request.signal.throwIfAborted(); + throw new ScaleSetProviderReconciliationError(undefined, { cause: error }); } } @@ -497,7 +512,18 @@ export function validateProviderResult(result: ScaleSetReconcileResult, desiredR throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); } const record = value as Record; - const statuses = new Set(['converged', 'retained', 'retryable_error', 'non_retryable_error']); + const resultFields = new Set([ + 'status', + 'desiredRunners', + 'currentRunners', + 'needsRunnerInventory', + 'actions', + 'errors', + ]); + if (Object.keys(record).some((key) => !resultFields.has(key))) { + throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); + } + const statuses = new Set(['converged', 'retained', 'error']); if (!statuses.has(record.status as string)) { throw new ScaleSetProtocolError('scale-set compute provider returned an invalid status'); } @@ -511,6 +537,10 @@ export function validateProviderResult(result: ScaleSetReconcileResult, desiredR if (typeof actions !== 'object' || actions === null || Array.isArray(actions)) { throw new ScaleSetProtocolError('scale-set compute provider returned invalid actions'); } + const actionFields = new Set(['launched', 'terminated', 'retainedBusy', 'retainedUnknown']); + if (Object.keys(actions).some((key) => !actionFields.has(key))) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid action counts'); + } for (const key of ['launched', 'terminated', 'retainedBusy', 'retainedUnknown']) { if (!boundedCount((actions as Record)[key])) { throw new ScaleSetProtocolError('scale-set compute provider returned invalid action counts'); @@ -529,14 +559,15 @@ export function validateProviderResult(result: ScaleSetReconcileResult, desiredR 'remove_runner', 'terminate', ]); + const errorFields = new Set(['operation', 'code', 'runnerName', 'resourceId']); for (const error of record.errors) { if (typeof error !== 'object' || error === null || Array.isArray(error)) { throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); } const metadata = error as Record; if ( + Object.keys(metadata).some((key) => !errorFields.has(key)) || !operations.has(metadata.operation as string) || - typeof metadata.retryable !== 'boolean' || typeof metadata.code !== 'string' || !/^[A-Za-z][A-Za-z0-9._-]{0,127}$/.test(metadata.code) || !optionalBoundedMetadata(metadata.runnerName) || @@ -545,6 +576,24 @@ export function validateProviderResult(result: ScaleSetReconcileResult, desiredR throw new ScaleSetProtocolError('scale-set compute provider returned invalid error metadata'); } } + if ((record.status === 'error') !== record.errors.length > 0) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid error status'); + } + let expectedStatus: ScaleSetReconcileResult['status'] = 'converged'; + if (record.errors.length > 0 || (record.currentRunners as number) < desiredRunners) { + expectedStatus = 'error'; + } else if (record.needsRunnerInventory || (record.currentRunners as number) > desiredRunners) { + expectedStatus = 'retained'; + } + if (record.status !== expectedStatus) { + throw new ScaleSetProtocolError('scale-set compute provider returned invalid reconciliation status'); + } +} + +function throwIfProviderError(result: ScaleSetReconcileResult): void { + if (result.status === 'error') { + throw new ScaleSetProviderReconciliationError(result); + } } function boundedCount(value: unknown): value is number { @@ -564,7 +613,7 @@ function hasAsciiControlCharacter(value: string): boolean { function isFatalReconcilerError(error: unknown): boolean { if (error instanceof ScaleSetConfigurationError || error instanceof ScaleSetProtocolError) return true; - if (error instanceof ScaleSetProviderReconciliationError) return !error.retryable; + if (error instanceof ScaleSetProviderReconciliationError) return true; if (!isScaleSetHttpError(error)) return false; return error.status >= 400 && error.status < 500 && ![408, 409, 425, 429].includes(error.status); } From 308999261056dad41e6da81191fdad8b4e72ed4b Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 13:26:39 +0200 Subject: [PATCH 07/19] refactor(scale-set): remove orchestration tag plumbing --- .../aws/ec2/src/runners.d.ts | 3 -- .../aws/ec2/src/runners.test.ts | 42 ------------------- .../compute-providers/aws/ec2/src/runners.ts | 16 ------- .../ec2/src/scale-set/configuration.test.ts | 11 ----- .../aws/ec2/src/scale-set/inventory.ts | 1 - .../aws/ec2/src/scale-set/scale-up.test.ts | 22 +++++++--- .../aws/ec2/src/scale-set/scale-up.ts | 2 +- 7 files changed, 17 insertions(+), 80 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts index 51b5432270..e711ed5318 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.d.ts @@ -6,7 +6,6 @@ import { _InstanceType, Placement, FleetBlockDeviceMappingRequest, - type Tag, } from '@aws-sdk/client-ec2'; import type { ListRunnerFilters, RunnerSource, RunnerType } from '../../../core'; @@ -48,6 +47,4 @@ export interface RunnerInputParameters { tracingEnabled?: boolean; onDemandFailoverOnError?: string[]; useDedicatedHost?: boolean; - /** Orchestrator-owned tags applied to instances, volumes, and fleets. */ - orchestrationTags?: readonly Tag[]; } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts index 6699f0399c..dd80e7b4dd 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.test.ts @@ -423,29 +423,6 @@ describe('create runner', () => { }); }); - it.each([ - ['a provider-owned tag', [{ Key: 'ghr:Owner', Value: 'another-owner' }]], - [ - 'a duplicate orchestration tag', - [ - { Key: 'ghr:scale_set_id', Value: '42' }, - { Key: 'ghr:scale_set_id', Value: '43' }, - ], - ], - ])('rejects %s before creating a Fleet', async (_description, orchestrationTags) => { - await expect( - ec2Operations.create({ - ...createRunnerConfig(defaultRunnerConfig), - orchestrationTags, - }), - ).resolves.toEqual({ - instances: [], - failedInstanceCount: 1, - failureCodes: ['aws-name:Error'], - }); - expect(mockEC2Client).not.toHaveReceivedCommand(CreateFleetCommand); - }); - it('calls create fleet of 1 instance with the on-demand capacity', async () => { await ec2Operations.create( createRunnerConfig({ ...defaultRunnerConfig, capacityType: 'on-demand', allocationStrategy: 'lowest-price' }), @@ -1516,25 +1493,6 @@ describe('create runner with useDedicatedHost', () => { }); }); - it('passes orchestration tags to RunInstances resources', async () => { - const orchestrationTags = [ - { Key: 'ghr:environment', Value: 'unit-test' }, - { Key: 'ghr:scale_set_id', Value: '42' }, - ]; - - await ec2Operations.create({ - ...createRunnerConfig(dedicatedHostRunnerConfig), - orchestrationTags, - }); - - expect(mockEC2Client).toHaveReceivedCommandWith(RunInstancesCommand, { - TagSpecifications: [ - { ResourceType: 'instance', Tags: expect.arrayContaining(orchestrationTags) }, - { ResourceType: 'volume', Tags: expect.arrayContaining(orchestrationTags) }, - ], - }); - }); - it('creates multiple instances via RunInstances and preserves the caller source', async () => { mockEC2Client.on(RunInstancesCommand).resolves({ Instances: [{ InstanceId: 'i-dedicated-1' }, { InstanceId: 'i-dedicated-2' }], diff --git a/lambdas/libs/compute-providers/aws/ec2/src/runners.ts b/lambdas/libs/compute-providers/aws/ec2/src/runners.ts index ac16551a87..d746888a6f 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/runners.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/runners.ts @@ -27,24 +27,12 @@ import type { Ec2RunnerCreateResult, Ec2RunnerFailureCode } from './runner-creat import type { Ec2ListRunnerFilters, Ec2OverrideConfig, RunnerInputParameters } from './runners.d'; const logger = createChildLogger('runners'); -const BASE_RUNNER_TAG_KEYS = new Set(['ghr:Application', 'ghr:created_by', 'ghr:Type', 'ghr:Owner', 'ghr:trace_id']); interface Ec2Filter { Name: string; Values: string[]; } -function appendOrchestrationRunnerTags(tags: Tag[], orchestrationTags: readonly Tag[] | undefined): void { - const keys = new Set(BASE_RUNNER_TAG_KEYS); - for (const tag of orchestrationTags ?? []) { - if (!tag.Key || tag.Value === undefined || keys.has(tag.Key)) { - throw new Error(`Orchestration runner tag '${tag.Key ?? ''}' is invalid or duplicates a provider-owned tag`); - } - keys.add(tag.Key); - tags.push({ Key: tag.Key, Value: tag.Value }); - } -} - export interface Ec2RunnerRequestContext { readonly signal: AbortSignal | undefined; } @@ -563,8 +551,6 @@ async function createInstances( { Key: 'ghr:Type', Value: runnerParameters.runnerType }, { Key: 'ghr:Owner', Value: runnerParameters.runnerOwner }, ]; - appendOrchestrationRunnerTags(tags, runnerParameters.orchestrationTags); - if (runnerParameters.tracingEnabled) { const traceId = tracer.getRootXrayTraceId(); tags.push({ Key: 'ghr:trace_id', Value: traceId! }); @@ -649,8 +635,6 @@ async function createInstancesWithRunInstances( { Key: 'ghr:Type', Value: runnerParameters.runnerType }, { Key: 'ghr:Owner', Value: runnerParameters.runnerOwner }, ]; - appendOrchestrationRunnerTags(tags, runnerParameters.orchestrationTags); - if (runnerParameters.tracingEnabled) { const traceId = tracer.getRootXrayTraceId(); tags.push({ Key: 'ghr:trace_id', Value: traceId! }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts index d2e50b34d7..d6483f3204 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/configuration.test.ts @@ -1,7 +1,6 @@ import { describe, expect, it } from 'vitest'; import { parseEc2ScaleSetProviderConfig } from './configuration'; -import { EC2_SCALE_SET_ID_TAG } from './inventory'; import { config } from './test/fixtures'; describe('EC2 scale-set provider configuration', () => { @@ -27,14 +26,4 @@ describe('EC2 scale-set provider configuration', () => { ])('rejects invalid or unsupported values instead of forwarding them to AWS', (invalid) => { expect(() => parseEc2ScaleSetProviderConfig(invalid)).toThrow(); }); - - it('does not expose configurable EC2 ownership or lifecycle tags', () => { - expect(Object.keys(config)).not.toContain('orchestrationTags'); - expect(() => - parseEc2ScaleSetProviderConfig({ - ...config, - orchestrationTags: [{ Key: EC2_SCALE_SET_ID_TAG, Value: 'another-scale-set' }], - }), - ).toThrow("Unsupported EC2 scale-set configuration field 'configuration.orchestrationTags'"); - }); }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts index c3e6d52a14..c0bc21cec9 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts @@ -58,7 +58,6 @@ export function ownershipTags(input: CreateEc2ScaleSetProviderInput): Tag[] { { Key: EC2_RUNNER_CONFIG_TAG, Value: input.runnerConfigName }, { Key: EC2_SCALE_SET_ID_TAG, Value: String(input.scaleSetId) }, { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash(input.githubScope) }, - { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, ]; } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts index 13a62a0c6f..f0a962faa7 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts @@ -1,4 +1,9 @@ -import { CreateFleetCommand, DescribeInstancesCommand, TerminateInstancesCommand } from '@aws-sdk/client-ec2'; +import { + CreateFleetCommand, + CreateTagsCommand, + DescribeInstancesCommand, + TerminateInstancesCommand, +} from '@aws-sdk/client-ec2'; import { DeleteParameterCommand, GetParameterCommand, PutParameterCommand } from '@aws-sdk/client-ssm'; import { beforeEach, describe, expect, it, vi } from 'vitest'; @@ -37,17 +42,22 @@ describe('EC2 scale-set scale up', () => { ResourceType: 'instance', Tags: expect.arrayContaining([ { Key: 'ghr:created_by', Value: 'scale-set-service' }, - { Key: 'ghr:environment', Value: 'unit-test' }, { Key: 'ghr:Owner', Value: 'example' }, { Key: 'ghr:Type', Value: 'Org' }, - { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, - { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, - { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, - { Key: EC2_SCALE_SET_STATE_TAG, Value: 'provisioning' }, ]), }), ]), }); + expect(ec2Mock).toHaveReceivedCommandWith(CreateTagsCommand, { + Resources: [instanceId], + Tags: expect.arrayContaining([ + { Key: 'ghr:environment', Value: 'unit-test' }, + { Key: EC2_RUNNER_CONFIG_TAG, Value: 'linux' }, + { Key: EC2_SCALE_SET_ID_TAG, Value: '42' }, + { Key: EC2_GITHUB_SCOPE_HASH_TAG, Value: githubScopeHash }, + { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, + ]), + }); expect(ssmMock).toHaveReceivedCommandWith(PutParameterCommand, { Name: `${config.jitConfigParameterPath}/${instanceId}`, Value: 'sensitive-encoded-jit-configuration', diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts index 41c4f943ad..a0fc3b984b 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.ts @@ -169,6 +169,7 @@ async function configureLaunchedRunner( try { await runners.tag(instanceId, [ + ...ownershipTags(input), { Key: EC2_RUNNER_NAME_TAG, Value: jit.runnerName }, { Key: EC2_GITHUB_RUNNER_ID_TAG, Value: String(jit.runnerId) }, { Key: EC2_SCALE_SET_STATE_TAG, Value: 'publishing' }, @@ -230,7 +231,6 @@ export async function scaleUp( tracingEnabled: input.configuration.tracingEnabled, onDemandFailoverOnError: input.configuration.onDemandFailoverOnError, useDedicatedHost: input.configuration.useDedicatedHost, - orchestrationTags: ownershipTags(input), }); } catch (error) { throwIfAborted(request.signal, error); From de69e1d8cea5fddf19081bc9471e3b243968152f Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:01:27 +0200 Subject: [PATCH 08/19] fix(scale-set): expose configuration error details --- lambdas/services/scale-set/src/logger.test.ts | 16 ++++++++++++++++ lambdas/services/scale-set/src/logger.ts | 9 ++++++++- lambdas/services/scale-set/src/main.ts | 10 ++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lambdas/services/scale-set/src/logger.test.ts b/lambdas/services/scale-set/src/logger.test.ts index 0a98b76ec8..6e7c881aa7 100644 --- a/lambdas/services/scale-set/src/logger.test.ts +++ b/lambdas/services/scale-set/src/logger.test.ts @@ -1,4 +1,5 @@ import { logger, sanitizeLogAttributes } from './logger'; +import { ScaleSetConfigurationError } from './config'; describe('redacted structured logging', () => { it('redacts nested secrets and strips log-injection characters', () => { @@ -23,4 +24,19 @@ describe('redacted structured logging', () => { expect(JSON.parse(spy.mock.calls[0][0] as string)).toMatchObject({ level: 'error', event: 'failed' }); spy.mockRestore(); }); + + it('includes safe configuration error messages for diagnosis', () => { + expect( + sanitizeLogAttributes({ + error: new ScaleSetConfigurationError( + 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + ), + }), + ).toEqual({ + error: { + name: 'ScaleSetConfigurationError', + message: 'provide exactly one of SCALE_SET_CONTROLLER_MANIFEST or SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH', + }, + }); + }); }); diff --git a/lambdas/services/scale-set/src/logger.ts b/lambdas/services/scale-set/src/logger.ts index daf0597c68..188950c86a 100644 --- a/lambdas/services/scale-set/src/logger.ts +++ b/lambdas/services/scale-set/src/logger.ts @@ -1,5 +1,6 @@ const REDACTED = '[REDACTED]'; const SENSITIVE_KEY = /(authorization|credential|encodedjit|jitconfig|password|private.?key|secret|sessionid|token)/i; +const SAFE_ERROR_MESSAGE_NAMES = new Set(['ScaleSetConfigurationError']); const MAX_LOG_STRING_LENGTH = 1024; const MAX_LOG_DEPTH = 4; @@ -21,7 +22,13 @@ function sanitize(value: unknown, key: string, depth: number): unknown { if (value instanceof Error) { const status = 'status' in value && typeof value.status === 'number' ? value.status : undefined; const code = 'code' in value && typeof value.code === 'string' ? sanitizeString(value.code) : undefined; - return { name: sanitizeString(value.name), ...(status === undefined ? {} : { status }), ...(code ? { code } : {}) }; + const message = SAFE_ERROR_MESSAGE_NAMES.has(value.name) ? sanitizeString(value.message) : undefined; + return { + name: sanitizeString(value.name), + ...(message ? { message } : {}), + ...(status === undefined ? {} : { status }), + ...(code ? { code } : {}), + }; } if (Array.isArray(value)) return value.slice(0, 100).map((item) => sanitize(item, key, depth + 1)); if (typeof value === 'object') { diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts index fb5425a4b7..942deed5e1 100644 --- a/lambdas/services/scale-set/src/main.ts +++ b/lambdas/services/scale-set/src/main.ts @@ -12,8 +12,18 @@ import { createDefaultControllerManifestLoader, defaultParameterStore } from './ import { abortableSleep, TtlScaleSetRunnerInventoryCache, type ScaleSetReconcilerDependencies } from './reconciler'; async function main(): Promise { + logger.info('scale_set_controller_configuration_loading', { + manifestConfigured: Boolean(process.env.SCALE_SET_CONTROLLER_MANIFEST?.trim()), + groupConfigConfigured: Boolean(process.env.SCALE_SET_CONTROLLER_GROUP_CONFIG_PATH?.trim()), + }); const serviceConfig = parseScaleSetServiceConfig(process.env); const manifest = await createDefaultControllerManifestLoader().load(serviceConfig); + logger.info('scale_set_controller_manifest_loaded', { + groupName: manifest.groupName, + revision: manifest.revision, + reconcilerCount: manifest.reconcilers.length, + runnerConfigNames: manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); const computeProviders = createScaleSetComputeProviderRegistry(); const githubHttp = createScaleSetGitHubHttp(); const dependencies: ScaleSetReconcilerDependencies = { From 9f8a74e30e92d20dc1eafd60a11c42ecbfdde19e Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:12:38 +0200 Subject: [PATCH 09/19] feat(scale-set): resolve GitHub IDs from names --- lambdas/services/scale-set/README.md | 6 +- lambdas/services/scale-set/src/config.test.ts | 21 ++++ lambdas/services/scale-set/src/config.ts | 45 ++++++--- lambdas/services/scale-set/src/main.ts | 2 +- .../services/scale-set/src/reconciler.test.ts | 39 +++++++- lambdas/services/scale-set/src/reconciler.ts | 96 +++++++++++++++---- 6 files changed, 175 insertions(+), 34 deletions(-) diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md index 38fadad978..55d7c92fbc 100644 --- a/lambdas/services/scale-set/README.md +++ b/lambdas/services/scale-set/README.md @@ -30,8 +30,8 @@ The service reads every direct child under the SSM path with paginated `GetParam "schemaVersion": 1, "runnerConfigName": "linux-x64", "githubConfigUrl": "https://github.com/example", - "scaleSetId": 123, - "expectedScaleSetName": "linux-x64", + "scaleSetName": "linux-x64", + "runnerGroupName": "self-hosted-linux", "expectedRunnerGroupId": null, "minRunners": 0, "maxRunners": 20, @@ -64,7 +64,7 @@ The service reads every direct child under the SSM path with paginated `GetParam } ``` -Optional fields are `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. `expectedRunnerGroupId` can be omitted or null. Before opening a session, the reconciler fetches the configured scale-set ID and verifies its expected name and, when supplied, runner-group ID. +`scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. The service resolves both numeric IDs through the configured GitHub Actions service endpoint at startup; `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. If the named scale set does not exist, startup fails with a configuration error; this service does not create GitHub scale sets implicitly. Optional fields are `scaleSetId`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. GitHub App values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. diff --git a/lambdas/services/scale-set/src/config.test.ts b/lambdas/services/scale-set/src/config.test.ts index 6dfef9c445..f0139e9155 100644 --- a/lambdas/services/scale-set/src/config.test.ts +++ b/lambdas/services/scale-set/src/config.test.ts @@ -121,6 +121,27 @@ describe('parseScaleSetControllerManifest', () => { }); }); + it('accepts a runner-group name for runtime ID resolution', () => { + expect( + parseScaleSetReconcilerConfig(runnerConfig({ runnerGroupName: 'self-hosted-linux' }), 0, 'group'), + ).toMatchObject({ runnerGroupName: 'self-hosted-linux' }); + }); + + it('accepts scaleSetName without a GitHub-generated scale-set ID', () => { + const parsed = parseScaleSetReconcilerConfig( + runnerConfig({ + scaleSetName: 'linux-x64', + expectedScaleSetName: undefined, + runnerGroupName: 'self-hosted-linux', + scaleSetId: undefined, + }), + 0, + 'group', + ); + expect(parsed).toMatchObject({ scaleSetName: 'linux-x64', runnerGroupName: 'self-hosted-linux' }); + expect(parsed).not.toHaveProperty('scaleSetId'); + }); + it('bounds the derived session owner for maximum-length names', () => { const parsed = parseScaleSetReconcilerConfig( runnerConfig({ runnerConfigName: 'r'.repeat(128) }), diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts index 15e98df45f..ccd233c0b7 100644 --- a/lambdas/services/scale-set/src/config.ts +++ b/lambdas/services/scale-set/src/config.ts @@ -15,8 +15,9 @@ export interface GitHubAppParameterReferences { export interface ScaleSetReconcilerConfig { schemaVersion: 1; runnerConfigName: string; - scaleSetId: number; - expectedScaleSetName: string; + scaleSetId?: number; + scaleSetName: string; + runnerGroupName?: string; expectedRunnerGroupId?: number; githubConfigUrl: string; githubApp: GitHubAppParameterReferences; @@ -340,7 +341,9 @@ export function parseScaleSetReconcilerConfig( 'schemaVersion', 'runnerConfigName', 'scaleSetId', + 'scaleSetName', 'expectedScaleSetName', + 'runnerGroupName', 'expectedRunnerGroupId', 'githubConfigUrl', 'githubApp', @@ -377,14 +380,28 @@ export function parseScaleSetReconcilerConfig( record.expectedRunnerGroupId === undefined || record.expectedRunnerGroupId === null ? undefined : integer(record, 'expectedRunnerGroupId', path, 1, MAX_SCALE_SET_CAPACITY); + const runnerGroupName = + record.runnerGroupName === undefined + ? undefined + : validateScaleSetName(requiredString(record, 'runnerGroupName', path), `${path}.runnerGroupName`); + if (record.scaleSetName !== undefined && record.expectedScaleSetName !== undefined) { + throw new ScaleSetConfigurationError(`${path} must configure only one of scaleSetName or expectedScaleSetName`); + } + const scaleSetName = validateScaleSetName( + requiredString(record, record.scaleSetName === undefined ? 'expectedScaleSetName' : 'scaleSetName', path), + `${path}.scaleSetName`, + ); + const scaleSetId = + record.scaleSetId === undefined ? undefined : integer(record, 'scaleSetId', path, 1, MAX_SCALE_SET_CAPACITY); + if (scaleSetId === undefined && runnerGroupName === undefined) { + throw new ScaleSetConfigurationError(`${path}.runnerGroupName is required when scaleSetId is omitted`); + } return { schemaVersion: 1, runnerConfigName, - scaleSetId: integer(record, 'scaleSetId', path, 1, MAX_SCALE_SET_CAPACITY), - expectedScaleSetName: validateScaleSetName( - requiredString(record, 'expectedScaleSetName', path), - `${path}.expectedScaleSetName`, - ), + ...(scaleSetId === undefined ? {} : { scaleSetId }), + scaleSetName, + ...(runnerGroupName === undefined ? {} : { runnerGroupName }), ...(expectedRunnerGroupId === undefined ? {} : { expectedRunnerGroupId }), githubConfigUrl: validateGitHubConfigUrl( requiredString(record, 'githubConfigUrl', path), @@ -460,20 +477,24 @@ export function parseScaleSetControllerManifest(input: string | unknown): ScaleS export function validateUniqueReconcilers(reconcilers: readonly ScaleSetReconcilerConfig[]): void { const names = new Set(); - const scopedScaleSetIds = new Set(); + const scopedScaleSets = new Set(); for (const reconciler of reconcilers) { if (names.has(reconciler.runnerConfigName)) { throw new ScaleSetConfigurationError( `runner config ${JSON.stringify(reconciler.runnerConfigName)} is duplicated`, ); } - const scopedScaleSetId = `${reconciler.githubConfigUrl}\u0000${reconciler.scaleSetId}`; - if (scopedScaleSetIds.has(scopedScaleSetId)) { + const scopedScaleSet = [ + reconciler.githubConfigUrl, + reconciler.runnerGroupName ?? String(reconciler.expectedRunnerGroupId ?? ''), + reconciler.scaleSetName, + ].join('\u0000'); + if (scopedScaleSets.has(scopedScaleSet)) { throw new ScaleSetConfigurationError( - `scale set ID ${reconciler.scaleSetId} is duplicated within GitHub scope ${JSON.stringify(reconciler.githubConfigUrl)}`, + `scale set ${JSON.stringify(reconciler.scaleSetName)} is duplicated within GitHub scope ${JSON.stringify(reconciler.githubConfigUrl)}`, ); } names.add(reconciler.runnerConfigName); - scopedScaleSetIds.add(scopedScaleSetId); + scopedScaleSets.add(scopedScaleSet); } } diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts index 942deed5e1..e57c07a885 100644 --- a/lambdas/services/scale-set/src/main.ts +++ b/lambdas/services/scale-set/src/main.ts @@ -45,7 +45,7 @@ async function main(): Promise { systemInfo: { system: config.userAgent ?? 'github-aws-runners', version: '1', - scaleSetId: config.scaleSetId, + scaleSetId: config.scaleSetId ?? 0, subsystem: 'scale-set-controller', }, }), diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index f226431dbf..dddfd3234a 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -16,7 +16,7 @@ const config: ScaleSetReconcilerConfig = { schemaVersion: 1, runnerConfigName: 'linux', scaleSetId: 42, - expectedScaleSetName: 'linux', + scaleSetName: 'linux', githubConfigUrl: 'https://github.com/example', githubApp: { appIdParameterName: '/app/id', @@ -89,6 +89,13 @@ function fixture(options: { }; const client: ScaleSetReconcilerClient = { getRunnerScaleSetById: vi.fn().mockResolvedValue({ id: 42, name: 'linux' }), + getRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + getRunnerGroupByName: vi.fn().mockResolvedValue({ + id: 7, + name: 'runner-group', + size: 0, + isDefaultGroup: false, + }), createMessageSessionClient: vi.fn().mockResolvedValue(options.session as MessageSessionClient), generateJitRunnerConfig: vi.fn(), getGitHubRunner: vi.fn().mockResolvedValue({ id: 5, name: 'runner-5', status: 'online', busy: false }), @@ -96,6 +103,8 @@ function fixture(options: { listGitHubRunners: vi.fn().mockResolvedValue([]), listRunners: vi.fn().mockResolvedValue([]), removeRunner: vi.fn(), + systemInfo: { scaleSetId: 42 }, + setSystemInfo: vi.fn(), }; const dependencies: ScaleSetReconcilerDependencies = { createAccessTokenProvider: vi.fn().mockResolvedValue(async () => ({ token: 'not-a-real-token' })), @@ -114,6 +123,34 @@ function fixture(options: { } describe('ScaleSetReconciler', () => { + it('resolves the GitHub runner-group and scale-set IDs from their names', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ + session, + reconcile: vi.fn(async () => { + abort.abort(); + return result(); + }), + }); + vi.mocked(client.getRunnerScaleSet).mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }); + + await new ScaleSetReconciler( + { ...config, scaleSetId: undefined, runnerGroupName: 'runner-group', scaleSetName: 'linux' }, + serviceConfig, + dependencies, + ).run(abort.signal, reporter()); + + expect(client.getRunnerGroupByName).toHaveBeenCalledWith('runner-group', { signal: abort.signal }); + expect(client.getRunnerScaleSet).toHaveBeenCalledWith(7, 'linux', { signal: abort.signal }); + expect(client.setSystemInfo).toHaveBeenCalledWith(expect.objectContaining({ scaleSetId: 42 })); + }); + it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { const order: string[] = []; const abort = new AbortController(); diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index 0cacd6847f..23737e4b6a 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -34,11 +34,15 @@ export type ScaleSetReconcilerClient = Pick< | 'createMessageSessionClient' | 'generateJitRunnerConfig' | 'getGitHubRunner' + | 'getRunnerGroupByName' + | 'getRunnerScaleSet' | 'getRunnerScaleSetById' | 'getRunnerByName' | 'listGitHubRunners' | 'listRunners' | 'removeRunner' + | 'setSystemInfo' + | 'systemInfo' >; export interface ScaleSetReconcilerDependencies { @@ -107,6 +111,8 @@ export class ScaleSetReconciler { private readonly lifecycle = new Map(); private readonly lifecycleLimit: number; private inventory?: { expiresAt: number; value: Promise }; + private resolvedScaleSetId?: number; + private resolvedRunnerGroupId?: number; constructor( private readonly config: ScaleSetReconcilerConfig, @@ -123,14 +129,18 @@ export class ScaleSetReconciler { let provider: ScaleSetComputeProvider; let client: ScaleSetReconcilerClient; try { + const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); + client = this.dependencies.createClient(this.config, accessTokenProvider); + const resolved = await this.resolveScaleSet(client, signal); + this.resolvedScaleSetId = resolved.scaleSetId; + this.resolvedRunnerGroupId = resolved.runnerGroupId; + client.setSystemInfo({ ...client.systemInfo, scaleSetId: resolved.scaleSetId }); provider = this.dependencies.computeProviders.create(this.config.computeProvider.type, { runnerConfigName: this.config.runnerConfigName, - scaleSetId: this.config.scaleSetId, + scaleSetId: resolved.scaleSetId, githubScope: this.config.githubConfigUrl, configuration: this.config.computeProvider.configuration, }); - const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); - client = this.dependencies.createClient(this.config, accessTokenProvider); } catch (error) { status.markFailed(error); this.log('error', 'scale_set_reconciler_initialization_failed', { error }); @@ -142,17 +152,16 @@ export class ScaleSetReconciler { let session: MessageSessionClient | undefined; let madeProgress = false; try { - const configuredScaleSet = await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); + const configuredScaleSet = await client.getRunnerScaleSetById(this.scaleSetId, { signal }); if ( configuredScaleSet === null || - configuredScaleSet.id !== this.config.scaleSetId || - configuredScaleSet.name !== this.config.expectedScaleSetName || - (this.config.expectedRunnerGroupId !== undefined && - configuredScaleSet.runnerGroupId !== this.config.expectedRunnerGroupId) + configuredScaleSet.id !== this.scaleSetId || + configuredScaleSet.name !== this.config.scaleSetName || + (this.resolvedRunnerGroupId !== undefined && configuredScaleSet.runnerGroupId !== this.resolvedRunnerGroupId) ) { throw new ScaleSetConfigurationError('configured GitHub runner scale set identity does not match'); } - session = await client.createMessageSessionClient(this.config.scaleSetId, this.config.sessionOwner, { signal }); + session = await client.createMessageSessionClient(this.scaleSetId, this.config.sessionOwner, { signal }); status.markSessionReady(); this.log('info', 'scale_set_session_created'); let latestStatistics = session.session.statistics ?? undefined; @@ -217,6 +226,61 @@ export class ScaleSetReconciler { status.markStopping(); } + private async resolveScaleSet( + client: ScaleSetReconcilerClient, + signal: AbortSignal, + ): Promise<{ scaleSetId: number; runnerGroupId?: number }> { + let runnerGroupId = this.config.expectedRunnerGroupId; + if (this.config.runnerGroupName !== undefined) { + const runnerGroup = await client.getRunnerGroupByName(this.config.runnerGroupName, { signal }); + if (runnerGroupId !== undefined && runnerGroupId !== runnerGroup.id) { + throw new ScaleSetConfigurationError( + `runner group ${JSON.stringify(this.config.runnerGroupName)} resolved to ID ${runnerGroup.id}, expected ${runnerGroupId}`, + ); + } + runnerGroupId = runnerGroup.id; + this.log('info', 'scale_set_runner_group_resolved', { + runnerConfigName: this.config.runnerConfigName, + runnerGroupName: this.config.runnerGroupName, + runnerGroupId, + }); + } + + if (this.config.scaleSetId === undefined && runnerGroupId === undefined) { + throw new ScaleSetConfigurationError('runner group ID was not resolved'); + } + const configuredScaleSet = + this.config.scaleSetId === undefined + ? await client.getRunnerScaleSet(runnerGroupId as number, this.config.scaleSetName, { signal }) + : await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); + if (configuredScaleSet === null || configuredScaleSet.id === undefined) { + throw new ScaleSetConfigurationError( + `GitHub runner scale set ${JSON.stringify(this.config.scaleSetName)} was not found`, + ); + } + if ( + configuredScaleSet.name !== this.config.scaleSetName || + (runnerGroupId !== undefined && configuredScaleSet.runnerGroupId !== runnerGroupId) || + (this.config.scaleSetId !== undefined && configuredScaleSet.id !== this.config.scaleSetId) + ) { + throw new ScaleSetConfigurationError('configured GitHub runner scale set identity does not match'); + } + this.log('info', 'scale_set_resolved', { + runnerConfigName: this.config.runnerConfigName, + scaleSetName: this.config.scaleSetName, + scaleSetId: configuredScaleSet.id, + runnerGroupId, + }); + return { scaleSetId: configuredScaleSet.id, runnerGroupId }; + } + + private get scaleSetId(): number { + if (this.resolvedScaleSetId === undefined) { + throw new ScaleSetConfigurationError('scale set ID was not resolved'); + } + return this.resolvedScaleSetId; + } + private async reconcile( client: ScaleSetReconcilerClient, provider: ScaleSetComputeProvider, @@ -239,13 +303,13 @@ export class ScaleSetReconciler { }) => { const jit = await client.generateJitRunnerConfig( { name: runnerName, workFolder: this.config.workFolder }, - this.config.scaleSetId, + this.scaleSetId, { signal: callbackSignal ?? signal }, ); if ( jit.runner === null || jit.runner.name !== runnerName || - jit.runner.runnerScaleSetId !== this.config.scaleSetId || + jit.runner.runnerScaleSetId !== this.scaleSetId || !Number.isSafeInteger(jit.runner.id) || jit.runner.id <= 0 ) { @@ -278,7 +342,7 @@ export class ScaleSetReconciler { runner.id !== expected.runnerId || runner.name !== expected.runnerName || runner.runnerScaleSetId !== expected.scaleSetId || - expected.scaleSetId !== this.config.scaleSetId + expected.scaleSetId !== this.scaleSetId ) { return { status: 'retained_unknown' as const }; } @@ -374,7 +438,7 @@ export class ScaleSetReconciler { this.lifecycle.delete(runnerName); return; } - this.lifecycle.set(runnerName, { runnerId, runnerName, scaleSetId: this.config.scaleSetId, lifecycle }); + this.lifecycle.set(runnerName, { runnerId, runnerName, scaleSetId: this.scaleSetId, lifecycle }); while (this.lifecycle.size > this.lifecycleLimit) { const oldest = this.lifecycle.keys().next().value as string | undefined; if (oldest === undefined) break; @@ -439,9 +503,7 @@ export class ScaleSetReconciler { this.inventoryCacheKey(), async () => await client.listGitHubRunners({ signal }), ), - ]).then(([actionsRunners, githubRunners]) => - joinRunnerInventory(actionsRunners, githubRunners, this.config.scaleSetId), - ); + ]).then(([actionsRunners, githubRunners]) => joinRunnerInventory(actionsRunners, githubRunners, this.scaleSetId)); this.inventory = { expiresAt: Date.now() + SCALE_SET_INVENTORY_TTL_MS, value }; try { return await value; @@ -462,7 +524,7 @@ export class ScaleSetReconciler { private log(level: 'info' | 'warn' | 'error', event: string, attributes: Record = {}): void { this.dependencies.logger[level](event, { groupRunnerConfig: this.config.runnerConfigName, - scaleSetId: this.config.scaleSetId, + scaleSetId: this.resolvedScaleSetId, ...attributes, }); } From 2627ce09480c649777af70b6bdfffcc6a70b9596 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:24:31 +0200 Subject: [PATCH 10/19] feat(scale-set): discover GitHub app installation --- lambdas/services/scale-set/README.md | 6 +- lambdas/services/scale-set/src/config.ts | 15 ++- .../scale-set/src/credentials.test.ts | 45 +++++++ lambdas/services/scale-set/src/credentials.ts | 120 +++++++++++++++--- 4 files changed, 159 insertions(+), 27 deletions(-) diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md index 55d7c92fbc..df5f3f8f25 100644 --- a/lambdas/services/scale-set/README.md +++ b/lambdas/services/scale-set/README.md @@ -32,15 +32,13 @@ The service reads every direct child under the SSM path with paginated `GetParam "githubConfigUrl": "https://github.com/example", "scaleSetName": "linux-x64", "runnerGroupName": "self-hosted-linux", - "expectedRunnerGroupId": null, "minRunners": 0, "maxRunners": 20, "bootTimeoutMinutes": 10, "sslVerify": true, "githubApp": { "appIdParameterName": "/runners/github-app/id", - "privateKeyParameterName": "/runners/github-app/key", - "installationIdParameterName": "/runners/github-app/installation-id" + "privateKeyParameterName": "/runners/github-app/key" }, "computeProvider": { "type": "ec2", @@ -66,7 +64,7 @@ The service reads every direct child under the SSM path with paginated `GetParam `scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. The service resolves both numeric IDs through the configured GitHub Actions service endpoint at startup; `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. If the named scale set does not exist, startup fails with a configuration error; this service does not create GitHub scale sets implicitly. Optional fields are `scaleSetId`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. -GitHub App values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. +GitHub App ID and private-key values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. `installationIdParameterName` is optional; when it is absent or its parameter is not present, the service creates a short-lived App JWT and discovers the installation by matching the configured organization or enterprise account through `GET /app/installations`. This works with GitHub.com, GHES, and GitHub Enterprise Cloud data-residency API hosts derived from `githubConfigUrl`. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, App JWTs, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. `SCALE_SET_CONTROLLER_MANIFEST` is supported only as a bounded local/test convenience. It contains `{ "version": 1, "groupName": "...", "reconcilers": [...] }` and uses the same reconciler objects. diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts index ccd233c0b7..371c61b0c2 100644 --- a/lambdas/services/scale-set/src/config.ts +++ b/lambdas/services/scale-set/src/config.ts @@ -8,7 +8,7 @@ export type JsonValue = JsonPrimitive | JsonValue[] | { readonly [key: string]: export interface GitHubAppParameterReferences { appIdParameterName: string; - installationIdParameterName: string; + installationIdParameterName?: string; privateKeyParameterName: string; } @@ -299,15 +299,20 @@ function validateJsonValue(value: unknown, path: string, depth = 0, counter = { function parseGitHubApp(value: unknown, path: string): GitHubAppParameterReferences { const record = objectValue(value, path); exactKeys(record, ['appIdParameterName', 'installationIdParameterName', 'privateKeyParameterName'], path); + const installationIdParameterName = optionalString(record, 'installationIdParameterName', path); return { appIdParameterName: validateSsmParameterName( requiredString(record, 'appIdParameterName', path), `${path}.appIdParameterName`, ), - installationIdParameterName: validateSsmParameterName( - requiredString(record, 'installationIdParameterName', path), - `${path}.installationIdParameterName`, - ), + ...(installationIdParameterName === undefined + ? {} + : { + installationIdParameterName: validateSsmParameterName( + installationIdParameterName, + `${path}.installationIdParameterName`, + ), + }), privateKeyParameterName: validateSsmParameterName( requiredString(record, 'privateKeyParameterName', path), `${path}.privateKeyParameterName`, diff --git a/lambdas/services/scale-set/src/credentials.test.ts b/lambdas/services/scale-set/src/credentials.test.ts index 3910bd24d7..3aac8d28f1 100644 --- a/lambdas/services/scale-set/src/credentials.test.ts +++ b/lambdas/services/scale-set/src/credentials.test.ts @@ -77,6 +77,51 @@ describe('GitHub App credentials', () => { expect(secondAuth).toHaveBeenCalledTimes(1); }); + it('discovers the installation ID from the configured organization when SSM does not provide one', async () => { + const store: ParameterStore = { + get: vi.fn().mockResolvedValue( + new Map([ + ['/app/id', '123'], + ['/app/key', encodedKey('abc')], + ]), + ), + }; + const appAuth = vi.fn().mockResolvedValue({ token: 'app-jwt' }); + const installationAuth = vi + .fn() + .mockResolvedValue({ token: 'installation-token', expiresAt: '2099-01-01T00:00:00Z' }); + const fetchImplementation = vi.fn().mockResolvedValue( + new Response(JSON.stringify({ installations: [{ id: 456, account: { login: 'example' } }] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + authMocks.createAppAuth.mockReturnValueOnce(appAuth).mockReturnValueOnce(installationAuth); + + const provider = await createGitHubAppAccessTokenProvider( + { appIdParameterName: '/app/id', privateKeyParameterName: '/app/key' }, + 'https://github.com/example', + false, + store, + fetchImplementation, + ); + + await expect(provider()).resolves.toMatchObject({ token: 'installation-token' }); + await expect(provider()).resolves.toMatchObject({ token: 'installation-token' }); + expect(appAuth).toHaveBeenCalledWith({ type: 'app' }); + expect(installationAuth).toHaveBeenCalledWith({ type: 'installation', installationId: 456 }); + expect(appAuth).toHaveBeenCalledTimes(1); + expect(fetchImplementation).toHaveBeenCalledTimes(1); + expect(fetchImplementation).toHaveBeenCalledWith( + expect.objectContaining({ + href: 'https://api.github.com/app/installations?per_page=100&page=1', + }), + expect.objectContaining({ + headers: expect.objectContaining({ Authorization: 'Bearer app-jwt' }), + }), + ); + }); + it.each([ [new Map([['/app/id', '123']]), 'was not returned'], [ diff --git a/lambdas/services/scale-set/src/credentials.ts b/lambdas/services/scale-set/src/credentials.ts index 9e3a87be11..74a1006611 100644 --- a/lambdas/services/scale-set/src/credentials.ts +++ b/lambdas/services/scale-set/src/credentials.ts @@ -17,10 +17,19 @@ export interface ParameterStore { interface GitHubAppCredentials { appId: string; - installationId: number; + installationId?: number; privateKey: string; } +interface GitHubAppInstallation { + id?: unknown; + account?: { login?: unknown }; +} + +interface GitHubAppInstallationsResponse { + installations?: unknown; +} + const MAX_PRIVATE_KEY_BYTES = 64 * 1024; function requiredParameter(values: ReadonlyMap, name: string): string { @@ -54,22 +63,25 @@ export async function loadGitHubAppCredentials( references: GitHubAppParameterReferences, parameterStore: ParameterStore, ): Promise { - const values = await parameterStore.get([ - references.appIdParameterName, - references.installationIdParameterName, - references.privateKeyParameterName, - ]); + const names = [references.appIdParameterName, references.privateKeyParameterName]; + if (references.installationIdParameterName !== undefined) names.splice(1, 0, references.installationIdParameterName); + const values = await parameterStore.get(names); const appId = requiredParameter(values, references.appIdParameterName).trim(); if (!/^[A-Za-z0-9_-]{1,128}$/.test(appId)) { throw new ScaleSetConfigurationError('GitHub App ID parameter is invalid'); } - const installationIdRaw = requiredParameter(values, references.installationIdParameterName).trim(); - if (!/^\d+$/.test(installationIdRaw)) { - throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); - } - const installationId = Number(installationIdRaw); - if (!Number.isSafeInteger(installationId) || installationId <= 0) { - throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + let installationId: number | undefined; + if (references.installationIdParameterName !== undefined) { + const installationIdRaw = values.get(references.installationIdParameterName)?.trim(); + if (installationIdRaw !== undefined && installationIdRaw !== '') { + if (!/^\d+$/.test(installationIdRaw)) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + installationId = Number(installationIdRaw); + if (!Number.isSafeInteger(installationId) || installationId <= 0) { + throw new ScaleSetConfigurationError('GitHub App installation ID parameter must be a positive integer'); + } + } } return { appId, @@ -78,6 +90,58 @@ export async function loadGitHubAppCredentials( }; } +async function discoverGitHubAppInstallationId( + credentials: Pick, + target: string, + apiBaseUrl: string, + fetchImplementation: ScaleSetFetch, +): Promise { + const appRequest = request.defaults({ baseUrl: apiBaseUrl, request: { fetch: fetchImplementation } }); + const appAuth = createAppAuth({ + appId: credentials.appId, + privateKey: credentials.privateKey, + request: appRequest, + }); + const appAuthentication = await appAuth({ type: 'app' }); + for (let page = 1; page <= 100; page += 1) { + const url = new URL('/app/installations', `${apiBaseUrl}/`); + url.searchParams.set('per_page', '100'); + url.searchParams.set('page', String(page)); + const response = await fetchImplementation(url, { + headers: { + Accept: 'application/vnd.github+json', + Authorization: `Bearer ${appAuthentication.token}`, + 'User-Agent': 'github-aws-runners/scale-set-controller', + }, + }); + if (!response.ok) { + throw new ScaleSetConfigurationError(`GitHub App installation discovery failed with HTTP ${response.status}`); + } + let payload: GitHubAppInstallationsResponse; + try { + payload = (await response.json()) as GitHubAppInstallationsResponse; + } catch (error) { + throw new ScaleSetConfigurationError('GitHub App installation discovery returned invalid JSON', { cause: error }); + } + if (!Array.isArray(payload.installations)) { + throw new ScaleSetConfigurationError('GitHub App installation discovery returned an invalid response'); + } + for (const value of payload.installations as unknown[]) { + if (typeof value !== 'object' || value === null || Array.isArray(value)) continue; + const candidate = value as GitHubAppInstallation; + if ( + Number.isSafeInteger(candidate.id) && + typeof candidate.account?.login === 'string' && + candidate.account.login.toLowerCase() === target.toLowerCase() + ) { + return candidate.id as number; + } + } + if (payload.installations.length < 100) break; + } + throw new ScaleSetConfigurationError(`GitHub App is not installed for ${JSON.stringify(target)}`); +} + export async function createGitHubAppAccessTokenProvider( references: GitHubAppParameterReferences, githubConfigUrl: string, @@ -94,26 +158,46 @@ export async function createGitHubAppAccessTokenProvider( auth: ReturnType; } | undefined; + let discovered: + | { + fingerprint: string; + installationId: number; + } + | undefined; return async () => { // Reload references for rotation visibility, but preserve the Octokit auth // instance while credentials are unchanged so its installation-token cache // remains effective. const credentials = await loadGitHubAppCredentials(references, parameterStore); - const fingerprint = createHash('sha256') + const credentialFingerprint = createHash('sha256') .update(credentials.appId) .update('\u0000') - .update(String(credentials.installationId)) - .update('\u0000') .update(credentials.privateKey) .digest('base64url'); + const target = parsedConfig.organization ?? parsedConfig.enterprise; + if (credentials.installationId === undefined && target === undefined) { + throw new ScaleSetConfigurationError( + 'GitHub App installation discovery requires an organization or enterprise URL', + ); + } + let installationId = credentials.installationId; + if (installationId === undefined) { + if (discovered?.fingerprint === credentialFingerprint) { + installationId = discovered.installationId; + } else { + installationId = await discoverGitHubAppInstallationId(credentials, target!, apiBaseUrl, fetchImplementation); + discovered = { fingerprint: credentialFingerprint, installationId }; + } + } + const fingerprint = `${credentialFingerprint}\u0000${installationId}`; if (cached?.fingerprint !== fingerprint) { cached = { fingerprint, - installationId: credentials.installationId, + installationId, auth: createAppAuth({ appId: credentials.appId, - installationId: credentials.installationId, + installationId, privateKey: credentials.privateKey, request: request.defaults({ baseUrl: apiBaseUrl, request: { fetch: fetchImplementation } }), }), From 7a8edb4b3934e442a9ac3a5fffc7160ddc0c9285 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:28:27 +0200 Subject: [PATCH 11/19] feat(scale-set): cache resolved IDs in SSM --- lambdas/services/scale-set/README.md | 3 +- lambdas/services/scale-set/src/config.ts | 10 ++++++ .../scale-set/src/credentials.test.ts | 5 ++- lambdas/services/scale-set/src/credentials.ts | 4 +++ lambdas/services/scale-set/src/main.ts | 1 + .../services/scale-set/src/parameter-store.ts | 14 +++++++-- .../services/scale-set/src/reconciler.test.ts | 10 +++++- lambdas/services/scale-set/src/reconciler.ts | 31 ++++++++++++++++++- 8 files changed, 72 insertions(+), 6 deletions(-) diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md index df5f3f8f25..37248d8f32 100644 --- a/lambdas/services/scale-set/README.md +++ b/lambdas/services/scale-set/README.md @@ -32,6 +32,7 @@ The service reads every direct child under the SSM path with paginated `GetParam "githubConfigUrl": "https://github.com/example", "scaleSetName": "linux-x64", "runnerGroupName": "self-hosted-linux", + "runnerGroupIdParameterName": "/runners/github-app/runner-group/self-hosted-linux", "minRunners": 0, "maxRunners": 20, "bootTimeoutMinutes": 10, @@ -62,7 +63,7 @@ The service reads every direct child under the SSM path with paginated `GetParam } ``` -`scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. The service resolves both numeric IDs through the configured GitHub Actions service endpoint at startup; `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. If the named scale set does not exist, startup fails with a configuration error; this service does not create GitHub scale sets implicitly. Optional fields are `scaleSetId`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. +`scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. `runnerGroupIdParameterName` is an optional SSM cache path. When present, the service reads the runner-group ID from that parameter; if it is missing, the service resolves the name through the configured GitHub Actions service endpoint and writes the ID back as a non-secret `String` parameter with overwrite enabled. The service resolves the scale-set ID from the group and scale-set names; `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. If the named scale set does not exist, startup fails with a configuration error; this service does not create GitHub scale sets implicitly. Optional fields are `scaleSetId`, `runnerGroupIdParameterName`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. GitHub App ID and private-key values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. `installationIdParameterName` is optional; when it is absent or its parameter is not present, the service creates a short-lived App JWT and discovers the installation by matching the configured organization or enterprise account through `GET /app/installations`. This works with GitHub.com, GHES, and GitHub Enterprise Cloud data-residency API hosts derived from `githubConfigUrl`. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, App JWTs, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts index 371c61b0c2..0f048dbe6c 100644 --- a/lambdas/services/scale-set/src/config.ts +++ b/lambdas/services/scale-set/src/config.ts @@ -18,6 +18,7 @@ export interface ScaleSetReconcilerConfig { scaleSetId?: number; scaleSetName: string; runnerGroupName?: string; + runnerGroupIdParameterName?: string; expectedRunnerGroupId?: number; githubConfigUrl: string; githubApp: GitHubAppParameterReferences; @@ -349,6 +350,7 @@ export function parseScaleSetReconcilerConfig( 'scaleSetName', 'expectedScaleSetName', 'runnerGroupName', + 'runnerGroupIdParameterName', 'expectedRunnerGroupId', 'githubConfigUrl', 'githubApp', @@ -389,6 +391,13 @@ export function parseScaleSetReconcilerConfig( record.runnerGroupName === undefined ? undefined : validateScaleSetName(requiredString(record, 'runnerGroupName', path), `${path}.runnerGroupName`); + const runnerGroupIdParameterName = + record.runnerGroupIdParameterName === undefined + ? undefined + : validateSsmParameterName( + requiredString(record, 'runnerGroupIdParameterName', path), + `${path}.runnerGroupIdParameterName`, + ); if (record.scaleSetName !== undefined && record.expectedScaleSetName !== undefined) { throw new ScaleSetConfigurationError(`${path} must configure only one of scaleSetName or expectedScaleSetName`); } @@ -407,6 +416,7 @@ export function parseScaleSetReconcilerConfig( ...(scaleSetId === undefined ? {} : { scaleSetId }), scaleSetName, ...(runnerGroupName === undefined ? {} : { runnerGroupName }), + ...(runnerGroupIdParameterName === undefined ? {} : { runnerGroupIdParameterName }), ...(expectedRunnerGroupId === undefined ? {} : { expectedRunnerGroupId }), githubConfigUrl: validateGitHubConfigUrl( requiredString(record, 'githubConfigUrl', path), diff --git a/lambdas/services/scale-set/src/credentials.test.ts b/lambdas/services/scale-set/src/credentials.test.ts index 3aac8d28f1..1dbc35d84c 100644 --- a/lambdas/services/scale-set/src/credentials.test.ts +++ b/lambdas/services/scale-set/src/credentials.test.ts @@ -31,6 +31,7 @@ describe('GitHub App credentials', () => { ['/app/key', encodedKey('abc')], ]), ), + put: vi.fn(), }; await expect(loadGitHubAppCredentials(references, store)).resolves.toMatchObject({ appId: '123', @@ -85,6 +86,7 @@ describe('GitHub App credentials', () => { ['/app/key', encodedKey('abc')], ]), ), + put: vi.fn(), }; const appAuth = vi.fn().mockResolvedValue({ token: 'app-jwt' }); const installationAuth = vi @@ -99,7 +101,7 @@ describe('GitHub App credentials', () => { authMocks.createAppAuth.mockReturnValueOnce(appAuth).mockReturnValueOnce(installationAuth); const provider = await createGitHubAppAccessTokenProvider( - { appIdParameterName: '/app/id', privateKeyParameterName: '/app/key' }, + references, 'https://github.com/example', false, store, @@ -112,6 +114,7 @@ describe('GitHub App credentials', () => { expect(installationAuth).toHaveBeenCalledWith({ type: 'installation', installationId: 456 }); expect(appAuth).toHaveBeenCalledTimes(1); expect(fetchImplementation).toHaveBeenCalledTimes(1); + expect(store.put).toHaveBeenCalledWith('/app/installation', '456'); expect(fetchImplementation).toHaveBeenCalledWith( expect.objectContaining({ href: 'https://api.github.com/app/installations?per_page=100&page=1', diff --git a/lambdas/services/scale-set/src/credentials.ts b/lambdas/services/scale-set/src/credentials.ts index 74a1006611..e6cf87bb01 100644 --- a/lambdas/services/scale-set/src/credentials.ts +++ b/lambdas/services/scale-set/src/credentials.ts @@ -13,6 +13,7 @@ import { ScaleSetConfigurationError, type GitHubAppParameterReferences } from '. export interface ParameterStore { get(names: readonly string[]): Promise>; + put?(name: string, value: string): Promise; } interface GitHubAppCredentials { @@ -188,6 +189,9 @@ export async function createGitHubAppAccessTokenProvider( } else { installationId = await discoverGitHubAppInstallationId(credentials, target!, apiBaseUrl, fetchImplementation); discovered = { fingerprint: credentialFingerprint, installationId }; + if (references.installationIdParameterName !== undefined && parameterStore.put !== undefined) { + await parameterStore.put(references.installationIdParameterName, String(installationId)); + } } } const fingerprint = `${credentialFingerprint}\u0000${installationId}`; diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts index e57c07a885..d7672224a6 100644 --- a/lambdas/services/scale-set/src/main.ts +++ b/lambdas/services/scale-set/src/main.ts @@ -50,6 +50,7 @@ async function main(): Promise { }, }), logger, + parameterStore: defaultParameterStore, sleep: abortableSleep, random: Math.random, closeSignal: AbortSignal.timeout, diff --git a/lambdas/services/scale-set/src/parameter-store.ts b/lambdas/services/scale-set/src/parameter-store.ts index ce07141bd7..3df49818b8 100644 --- a/lambdas/services/scale-set/src/parameter-store.ts +++ b/lambdas/services/scale-set/src/parameter-store.ts @@ -1,6 +1,6 @@ -import { GetParametersByPathCommand, SSMClient } from '@aws-sdk/client-ssm'; +import { GetParametersByPathCommand, PutParameterCommand, SSMClient } from '@aws-sdk/client-ssm'; -import { getParameters } from '@aws-github-runner/aws-ssm-util'; +import { getParameters, ssmClient } from '@aws-github-runner/aws-ssm-util'; import { MAX_MANIFEST_BYTES, @@ -20,6 +20,16 @@ const MAX_GROUP_BYTES = 4 * 1024 * 1024; export const defaultParameterStore: ParameterStore = { get: async (names) => await getParameters([...names]), + put: async (name, value) => { + await ssmClient().send( + new PutParameterCommand({ + Name: name, + Value: value, + Type: 'String', + Overwrite: true, + }), + ); + }, }; export interface ControllerManifestLoader { diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index dddfd3234a..8062c04193 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -118,6 +118,7 @@ function fixture(options: { random: () => 0, closeSignal: () => new AbortController().signal, runnerInventory: new TtlScaleSetRunnerInventoryCache(), + parameterStore: { get: vi.fn().mockResolvedValue(new Map()), put: vi.fn() }, }; return { client, computeProvider, dependencies }; } @@ -141,7 +142,13 @@ describe('ScaleSetReconciler', () => { vi.mocked(client.getRunnerScaleSet).mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }); await new ScaleSetReconciler( - { ...config, scaleSetId: undefined, runnerGroupName: 'runner-group', scaleSetName: 'linux' }, + { + ...config, + scaleSetId: undefined, + runnerGroupName: 'runner-group', + runnerGroupIdParameterName: '/runner/group-id', + scaleSetName: 'linux', + }, serviceConfig, dependencies, ).run(abort.signal, reporter()); @@ -149,6 +156,7 @@ describe('ScaleSetReconciler', () => { expect(client.getRunnerGroupByName).toHaveBeenCalledWith('runner-group', { signal: abort.signal }); expect(client.getRunnerScaleSet).toHaveBeenCalledWith(7, 'linux', { signal: abort.signal }); expect(client.setSystemInfo).toHaveBeenCalledWith(expect.objectContaining({ scaleSetId: 42 })); + expect(dependencies.parameterStore.put).toHaveBeenCalledWith('/runner/group-id', '7'); }); it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index 23737e4b6a..fafba33a90 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -19,6 +19,7 @@ import type { } from '@aws-github-runner/compute-providers/scale-set'; import { ScaleSetConfigurationError, type ScaleSetReconcilerConfig, type ScaleSetServiceConfig } from './config'; +import type { ParameterStore } from './credentials'; import type { ScaleSetReconcilerStatusReporter } from './health'; import type { ScaleSetLogger } from './logger'; @@ -54,6 +55,7 @@ export interface ScaleSetReconcilerDependencies { random(): number; closeSignal(timeoutMs: number): AbortSignal; runnerInventory: ScaleSetRunnerInventoryCache; + parameterStore: ParameterStore; } export interface ScaleSetRunnerInventoryCache { @@ -232,13 +234,20 @@ export class ScaleSetReconciler { ): Promise<{ scaleSetId: number; runnerGroupId?: number }> { let runnerGroupId = this.config.expectedRunnerGroupId; if (this.config.runnerGroupName !== undefined) { - const runnerGroup = await client.getRunnerGroupByName(this.config.runnerGroupName, { signal }); + const cachedRunnerGroupId = await this.loadCachedRunnerGroupId(); + const runnerGroup = + cachedRunnerGroupId === undefined + ? await client.getRunnerGroupByName(this.config.runnerGroupName, { signal }) + : { id: cachedRunnerGroupId }; if (runnerGroupId !== undefined && runnerGroupId !== runnerGroup.id) { throw new ScaleSetConfigurationError( `runner group ${JSON.stringify(this.config.runnerGroupName)} resolved to ID ${runnerGroup.id}, expected ${runnerGroupId}`, ); } runnerGroupId = runnerGroup.id; + if (cachedRunnerGroupId === undefined && this.config.runnerGroupIdParameterName !== undefined) { + await this.dependencies.parameterStore.put?.(this.config.runnerGroupIdParameterName, String(runnerGroupId)); + } this.log('info', 'scale_set_runner_group_resolved', { runnerConfigName: this.config.runnerConfigName, runnerGroupName: this.config.runnerGroupName, @@ -274,6 +283,26 @@ export class ScaleSetReconciler { return { scaleSetId: configuredScaleSet.id, runnerGroupId }; } + private async loadCachedRunnerGroupId(): Promise { + const parameterName = this.config.runnerGroupIdParameterName; + if (parameterName === undefined) return undefined; + const values = await this.dependencies.parameterStore.get([parameterName]); + const raw = values.get(parameterName)?.trim(); + if (raw === undefined || raw === '') return undefined; + if (!/^\d+$/.test(raw)) { + throw new ScaleSetConfigurationError( + `runner group ID parameter ${JSON.stringify(parameterName)} must contain a positive integer`, + ); + } + const id = Number(raw); + if (!Number.isSafeInteger(id) || id <= 0) { + throw new ScaleSetConfigurationError( + `runner group ID parameter ${JSON.stringify(parameterName)} must contain a positive integer`, + ); + } + return id; + } + private get scaleSetId(): number { if (this.resolvedScaleSetId === undefined) { throw new ScaleSetConfigurationError('scale set ID was not resolved'); From 7220eb22f69286da496010e2139d2712ff2a8179 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:30:51 +0200 Subject: [PATCH 12/19] fix(scale-set): parse GitHub installation list --- lambdas/services/scale-set/src/credentials.test.ts | 2 +- lambdas/services/scale-set/src/credentials.ts | 14 +++++--------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/lambdas/services/scale-set/src/credentials.test.ts b/lambdas/services/scale-set/src/credentials.test.ts index 1dbc35d84c..4536a8e582 100644 --- a/lambdas/services/scale-set/src/credentials.test.ts +++ b/lambdas/services/scale-set/src/credentials.test.ts @@ -93,7 +93,7 @@ describe('GitHub App credentials', () => { .fn() .mockResolvedValue({ token: 'installation-token', expiresAt: '2099-01-01T00:00:00Z' }); const fetchImplementation = vi.fn().mockResolvedValue( - new Response(JSON.stringify({ installations: [{ id: 456, account: { login: 'example' } }] }), { + new Response(JSON.stringify([{ id: 456, account: { login: 'example' } }]), { status: 200, headers: { 'content-type': 'application/json' }, }), diff --git a/lambdas/services/scale-set/src/credentials.ts b/lambdas/services/scale-set/src/credentials.ts index e6cf87bb01..f1178efff2 100644 --- a/lambdas/services/scale-set/src/credentials.ts +++ b/lambdas/services/scale-set/src/credentials.ts @@ -27,10 +27,6 @@ interface GitHubAppInstallation { account?: { login?: unknown }; } -interface GitHubAppInstallationsResponse { - installations?: unknown; -} - const MAX_PRIVATE_KEY_BYTES = 64 * 1024; function requiredParameter(values: ReadonlyMap, name: string): string { @@ -118,16 +114,16 @@ async function discoverGitHubAppInstallationId( if (!response.ok) { throw new ScaleSetConfigurationError(`GitHub App installation discovery failed with HTTP ${response.status}`); } - let payload: GitHubAppInstallationsResponse; + let payload: unknown; try { - payload = (await response.json()) as GitHubAppInstallationsResponse; + payload = await response.json(); } catch (error) { throw new ScaleSetConfigurationError('GitHub App installation discovery returned invalid JSON', { cause: error }); } - if (!Array.isArray(payload.installations)) { + if (!Array.isArray(payload)) { throw new ScaleSetConfigurationError('GitHub App installation discovery returned an invalid response'); } - for (const value of payload.installations as unknown[]) { + for (const value of payload as unknown[]) { if (typeof value !== 'object' || value === null || Array.isArray(value)) continue; const candidate = value as GitHubAppInstallation; if ( @@ -138,7 +134,7 @@ async function discoverGitHubAppInstallationId( return candidate.id as number; } } - if (payload.installations.length < 100) break; + if (payload.length < 100) break; } throw new ScaleSetConfigurationError(`GitHub App is not installed for ${JSON.stringify(target)}`); } From b13b7464f1cdcfe2af36b7fafbcfbc9b307dfaff Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:38:29 +0200 Subject: [PATCH 13/19] feat(scale-set): register missing GitHub scale sets --- lambdas/services/scale-set/README.md | 2 +- .../services/scale-set/src/reconciler.test.ts | 35 +++++++++++++++++++ lambdas/services/scale-set/src/reconciler.ts | 25 ++++++++++++- 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md index 37248d8f32..48b282bf0e 100644 --- a/lambdas/services/scale-set/README.md +++ b/lambdas/services/scale-set/README.md @@ -63,7 +63,7 @@ The service reads every direct child under the SSM path with paginated `GetParam } ``` -`scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. `runnerGroupIdParameterName` is an optional SSM cache path. When present, the service reads the runner-group ID from that parameter; if it is missing, the service resolves the name through the configured GitHub Actions service endpoint and writes the ID back as a non-secret `String` parameter with overwrite enabled. The service resolves the scale-set ID from the group and scale-set names; `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. If the named scale set does not exist, startup fails with a configuration error; this service does not create GitHub scale sets implicitly. Optional fields are `scaleSetId`, `runnerGroupIdParameterName`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. +`scaleSetName` and `runnerGroupName` are the GitHub names supplied by the operator. `runnerGroupIdParameterName` is an optional SSM cache path. When present, the service reads the runner-group ID from that parameter; if it is missing, the service resolves the name through the configured GitHub Actions service endpoint and writes the ID back as a non-secret `String` parameter with overwrite enabled. The service resolves the scale-set ID from the group and scale-set names; if the named scale set does not exist, it registers it in the resolved runner group and uses the ID returned by GitHub. `scaleSetId` is only an optional legacy pin for an already-known ID. `expectedRunnerGroupId` can be omitted or null; when supplied, it is treated as an additional consistency check. Optional fields are `scaleSetId`, `runnerGroupIdParameterName`, `expectedRunnerGroupId`, `sessionOwner`, `workFolder`, `forceGhes`, `sslVerify`, and `userAgent`. `sslVerify` defaults to `true`; when false, the service uses a reconciler-scoped Undici dispatcher for both GitHub App token and scale-set requests without changing `NODE_TLS_REJECT_UNAUTHORIZED` or the global dispatcher. `userAgent` becomes the `system` identity inside the required structured scale-set protocol User-Agent rather than replacing that header. `bootTimeoutMinutes` defaults to `10`; it is orchestration-owned and is passed to the selected compute provider on every reconciliation. GitHub App ID and private-key values are reloaded from SSM whenever an installation token is requested, so key rotation does not require a task restart. `installationIdParameterName` is optional; when it is absent or its parameter is not present, the service creates a short-lived App JWT and discovers the installation by matching the configured organization or enterprise account through `GET /app/installations`. This works with GitHub.com, GHES, and GitHub Enterprise Cloud data-residency API hosts derived from `githubConfigUrl`. A SHA-256 credential fingerprint keeps the same Octokit auth instance—and its token cache—while the values remain unchanged, and replaces it after rotation. Private keys, installation tokens, App JWTs, message-session tokens, message bodies, and JIT configurations are never accepted as manifest values and are redacted from logs. diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index 8062c04193..723cbe8e87 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -90,6 +90,7 @@ function fixture(options: { const client: ScaleSetReconcilerClient = { getRunnerScaleSetById: vi.fn().mockResolvedValue({ id: 42, name: 'linux' }), getRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), + createRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), getRunnerGroupByName: vi.fn().mockResolvedValue({ id: 7, name: 'runner-group', @@ -159,6 +160,40 @@ describe('ScaleSetReconciler', () => { expect(dependencies.parameterStore.put).toHaveBeenCalledWith('/runner/group-id', '7'); }); + it('registers a missing scale set in the resolved runner group', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn().mockResolvedValue(message()), + deleteMessage: vi.fn(), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ + session, + reconcile: vi.fn(async () => { + abort.abort(); + return result(); + }), + }); + vi.mocked(client.getRunnerScaleSet).mockResolvedValueOnce(null); + + await new ScaleSetReconciler( + { ...config, scaleSetId: undefined, runnerGroupName: 'runner-group', scaleSetName: 'linux' }, + serviceConfig, + dependencies, + ).run(abort.signal, reporter()); + + expect(client.createRunnerScaleSet).toHaveBeenCalledWith( + { + name: 'linux', + runnerGroupId: 7, + labels: [{ name: 'linux' }], + runnerSetting: {}, + }, + { signal: abort.signal }, + ); + }); + it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { const order: string[] = []; const abort = new AbortController(); diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index fafba33a90..b8478d7954 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -42,6 +42,7 @@ export type ScaleSetReconcilerClient = Pick< | 'listGitHubRunners' | 'listRunners' | 'removeRunner' + | 'createRunnerScaleSet' | 'setSystemInfo' | 'systemInfo' >; @@ -258,10 +259,32 @@ export class ScaleSetReconciler { if (this.config.scaleSetId === undefined && runnerGroupId === undefined) { throw new ScaleSetConfigurationError('runner group ID was not resolved'); } - const configuredScaleSet = + let configuredScaleSet = this.config.scaleSetId === undefined ? await client.getRunnerScaleSet(runnerGroupId as number, this.config.scaleSetName, { signal }) : await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); + if (configuredScaleSet === null && runnerGroupId !== undefined && this.config.scaleSetId === undefined) { + this.log('info', 'scale_set_registering', { + runnerConfigName: this.config.runnerConfigName, + scaleSetName: this.config.scaleSetName, + runnerGroupId, + }); + try { + configuredScaleSet = await client.createRunnerScaleSet( + { + name: this.config.scaleSetName, + runnerGroupId, + labels: [{ name: this.config.scaleSetName }], + runnerSetting: {}, + }, + { signal }, + ); + } catch (error) { + const existingScaleSet = await client.getRunnerScaleSet(runnerGroupId, this.config.scaleSetName, { signal }); + if (existingScaleSet === null) throw error; + configuredScaleSet = existingScaleSet; + } + } if (configuredScaleSet === null || configuredScaleSet.id === undefined) { throw new ScaleSetConfigurationError( `GitHub runner scale set ${JSON.stringify(this.config.scaleSetName)} was not found`, From fdb5b7604566246cf01ca87a445430fdb255efac Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:54:25 +0200 Subject: [PATCH 14/19] feat(scale-set): log compute provider lifecycle --- .../services/scale-set/src/reconciler.test.ts | 8 +++++++ lambdas/services/scale-set/src/reconciler.ts | 21 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index 723cbe8e87..1d4c7b271d 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -158,6 +158,14 @@ describe('ScaleSetReconciler', () => { expect(client.getRunnerScaleSet).toHaveBeenCalledWith(7, 'linux', { signal: abort.signal }); expect(client.setSystemInfo).toHaveBeenCalledWith(expect.objectContaining({ scaleSetId: 42 })); expect(dependencies.parameterStore.put).toHaveBeenCalledWith('/runner/group-id', '7'); + expect(dependencies.logger.info).toHaveBeenCalledWith( + 'scale_set_compute_provider_created', + expect.objectContaining({ computeProviderType: 'ec2' }), + ); + expect(dependencies.logger.info).toHaveBeenCalledWith( + 'scale_set_compute_provider_reconcile_started', + expect.objectContaining({ computeProviderType: 'ec2', desiredRunners: 1, runnerInventoryComplete: false }), + ); }); it('registers a missing scale set in the resolved runner group', async () => { diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index b8478d7954..2c4e51163c 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -144,9 +144,15 @@ export class ScaleSetReconciler { githubScope: this.config.githubConfigUrl, configuration: this.config.computeProvider.configuration, }); + this.log('info', 'scale_set_compute_provider_created', { + computeProviderType: this.config.computeProvider.type, + }); } catch (error) { status.markFailed(error); - this.log('error', 'scale_set_reconciler_initialization_failed', { error }); + this.log('error', 'scale_set_reconciler_initialization_failed', { + computeProviderType: this.config.computeProvider.type, + error, + }); return; } @@ -446,6 +452,7 @@ export class ScaleSetReconciler { } } this.log('info', 'scale_set_reconciled', { + computeProviderType: this.config.computeProvider.type, desiredRunners, currentRunners: result.currentRunners, status: result.status, @@ -454,6 +461,7 @@ export class ScaleSetReconciler { }); if (result.status === 'retained') { this.log('warn', 'scale_set_capacity_retained', { + computeProviderType: this.config.computeProvider.type, desiredRunners, currentRunners: result.currentRunners, retainedBusy: result.actions.retainedBusy, @@ -467,10 +475,21 @@ export class ScaleSetReconciler { provider: ScaleSetComputeProvider, request: ScaleSetReconcileRequest, ): Promise { + this.log('info', 'scale_set_compute_provider_reconcile_started', { + computeProviderType: this.config.computeProvider.type, + desiredRunners: request.desiredRunners, + runnerInventoryComplete: request.runnerInventoryComplete, + }); try { return await provider.reconcile(request); } catch (error) { request.signal.throwIfAborted(); + this.log('error', 'scale_set_compute_provider_reconcile_failed', { + computeProviderType: this.config.computeProvider.type, + desiredRunners: request.desiredRunners, + runnerInventoryComplete: request.runnerInventoryComplete, + error, + }); throw new ScaleSetProviderReconciliationError(undefined, { cause: error }); } } From d684e5e13b7fbee21f127d32090e16cea19aefc7 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 14:59:24 +0200 Subject: [PATCH 15/19] fix(scale-set): include endpoint in HTTP failure logs --- lambdas/services/scale-set/src/reconciler.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index 2c4e51163c..31a95072b1 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -151,6 +151,7 @@ export class ScaleSetReconciler { status.markFailed(error); this.log('error', 'scale_set_reconciler_initialization_failed', { computeProviderType: this.config.computeProvider.type, + ...httpErrorLogAttributes(error), error, }); return; @@ -210,7 +211,10 @@ export class ScaleSetReconciler { if (signal.aborted) break; if (isFatalReconcilerError(error)) { status.markFailed(error); - this.log('error', 'scale_set_reconciler_failed', { error }); + this.log('error', 'scale_set_reconciler_failed', { + ...httpErrorLogAttributes(error), + error, + }); return; } consecutiveFailures = madeProgress ? 1 : consecutiveFailures + 1; @@ -751,6 +755,16 @@ function isFatalReconcilerError(error: unknown): boolean { return error.status >= 400 && error.status < 500 && ![408, 409, 425, 429].includes(error.status); } +function httpErrorLogAttributes(error: unknown): Record { + if (!isScaleSetHttpError(error)) return {}; + return { + requestMethod: error.method, + requestUrl: error.url, + requestStatus: error.status, + requestCode: error.code, + }; +} + function joinRunnerInventory( actionsRunners: readonly { id: number; name: string; runnerScaleSetId: number }[], githubRunners: readonly GitHubRunnerReference[], From 7f05bf3021b8a9eae6a56b7a22539678ddfc296f Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 15:17:35 +0200 Subject: [PATCH 16/19] fix(scale-set): recover from inventory lookup failures --- .../services/scale-set/src/reconciler.test.ts | 47 ++++++++++++++++++- lambdas/services/scale-set/src/reconciler.ts | 18 ++++++- 2 files changed, 62 insertions(+), 3 deletions(-) diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index 1d4c7b271d..7e28a09ee6 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -1,4 +1,8 @@ -import type { MessageSessionClient, RunnerScaleSetMessage } from '@aws-github-runner/github-actions-scale-set'; +import { + ScaleSetHttpError, + type MessageSessionClient, + type RunnerScaleSetMessage, +} from '@aws-github-runner/github-actions-scale-set'; import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '@aws-github-runner/compute-providers/scale-set'; import type { ScaleSetReconcilerConfig, ScaleSetServiceConfig } from './config'; @@ -299,6 +303,47 @@ describe('ScaleSetReconciler', () => { expect(client.listRunners).toHaveBeenCalledTimes(1); }); + it('retains capacity and retries after a runner inventory 404 during recovery', async () => { + const abort = new AbortController(); + const inventoryError = new ScaleSetHttpError({ + method: 'GET', + url: 'https://api.github.com/orgs/example/actions/runners', + status: 404, + statusText: 'Not Found', + headers: new Headers(), + responseBody: '', + }); + const reconcile = vi.fn().mockResolvedValue( + result({ + status: 'retained', + currentRunners: 1, + needsRunnerInventory: true, + actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, + }), + ); + const session = { + session: { statistics: message().statistics }, + getMessage: vi.fn(async () => { + abort.abort(); + throw new DOMException('aborted', 'AbortError'); + }), + close: vi.fn(), + }; + const { client, dependencies } = fixture({ session, reconcile }); + vi.mocked(client.listGitHubRunners).mockRejectedValue(inventoryError); + const status = reporter(); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); + + expect(status.markFailed).not.toHaveBeenCalled(); + expect(reconcile).toHaveBeenCalledTimes(1); + expect(dependencies.logger.warn).toHaveBeenCalledWith( + 'scale_set_runner_inventory_unavailable', + expect.objectContaining({ requestMethod: 'GET', requestStatus: 404, requestCode: 'NOT_FOUND' }), + ); + expect(client.listGitHubRunners).toHaveBeenCalledTimes(1); + }); + it('rejects a provider that requests another inventory after the complete second pass', async () => { const abort = new AbortController(); const reconcile = vi.fn().mockResolvedValue( diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index 31a95072b1..fce47a04f5 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -439,7 +439,18 @@ export class ScaleSetReconciler { validateProviderResult(result, desiredRunners); throwIfProviderError(result); if (result.needsRunnerInventory) { - const inventory = await this.loadScaleSetInventory(client, signal); + let inventory: readonly GitHubScaleSetRunnerState[]; + try { + inventory = await this.loadScaleSetInventory(client, signal); + } catch (error) { + if (!isScaleSetHttpError(error) || error.status !== 404) throw error; + this.log('warn', 'scale_set_runner_inventory_unavailable', { + ...httpErrorLogAttributes(error), + error, + }); + this.logReconciliationResult(result, desiredRunners); + return; + } result = await this.reconcileProvider(provider, { desiredRunners, bootTimeoutMinutes: this.config.bootTimeoutMinutes, @@ -455,6 +466,10 @@ export class ScaleSetReconciler { ); } } + this.logReconciliationResult(result, desiredRunners); + } + + private logReconciliationResult(result: ScaleSetReconcileResult, desiredRunners: number): void { this.log('info', 'scale_set_reconciled', { computeProviderType: this.config.computeProvider.type, desiredRunners, @@ -471,7 +486,6 @@ export class ScaleSetReconciler { retainedBusy: result.actions.retainedBusy, retainedUnknown: result.actions.retainedUnknown, }); - return; } } From fae929ed03cdfea8c2ea953daf7e88e9b5843cca Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 15:36:34 +0200 Subject: [PATCH 17/19] feat(scale-set): add independent recovery janitor --- .../aws/ec2/src/scale-set/inventory.ts | 24 ++- .../aws/ec2/src/scale-set/provider.test.ts | 32 ++++ .../aws/ec2/src/scale-set/provider.ts | 8 +- .../aws/ec2/src/scale-set/scale-down.ts | 2 +- lambdas/libs/compute-providers/scale-set.ts | 2 + lambdas/services/scale-set/README.md | 48 +++++- lambdas/services/scale-set/src/config.test.ts | 19 ++- lambdas/services/scale-set/src/config.ts | 15 ++ lambdas/services/scale-set/src/controller.ts | 8 + lambdas/services/scale-set/src/main.ts | 36 +++++ .../services/scale-set/src/reconciler.test.ts | 34 +++- lambdas/services/scale-set/src/reconciler.ts | 150 +++++++++++++----- 12 files changed, 321 insertions(+), 57 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts index c0bc21cec9..aac37bd4c8 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts @@ -206,6 +206,23 @@ export function servingCapacity( const serving: OwnedEc2Runner[] = []; for (const runner of runners) { + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + if (request.recoveryOnly) { + if ( + githubState !== undefined && + (githubState.status === 'online' || githubState.status === 'offline') && + typeof githubState.busy === 'boolean' + ) { + // Recovery gets a fresh public GitHub status and may classify an + // offline, non-busy runner as removable. Unknown and busy identities + // remain in the scale-down classifier and are retained there. + serving.push(runner); + } else { + retainUnknown(state, runner.instanceId); + } + continue; + } + if (runner.scaleSetState !== 'config-published') { // An interrupted publication may already have been consumed. Preserve it, // but do not let it suppress replacement capacity indefinitely. @@ -213,7 +230,6 @@ export function servingCapacity( continue; } - const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); if (githubState !== undefined && isConfirmedServingState(githubState)) { serving.push(runner); continue; @@ -236,10 +252,12 @@ export function servingCapacity( return serving; } -export function isSafeScaleDownState(state: ScaleSetRunnerState): boolean { +export function isSafeScaleDownState(state: ScaleSetRunnerState, recoveryOnly = false): boolean { return ( (state.lifecycle === 'completed' && state.busy !== true) || - (state.lifecycle !== 'started' && state.status === 'online' && state.busy === false) + (state.lifecycle !== 'started' && + (state.status === 'online' || (recoveryOnly && state.status === 'offline')) && + state.busy === false) ); } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts index 67570584ac..5b17142e12 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts @@ -123,6 +123,38 @@ describe('EC2 scale-set provider orchestration', () => { expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); }); + it('recovery removes only exact idle runners and retains busy or unknown runners', async () => { + const idle = ownedInstance('i-idle', { runnerId: 100, runnerName: 'runner-i-idle' }); + const busy = ownedInstance('i-busy', { runnerId: 101, runnerName: 'runner-i-busy' }); + const unknown = ownedInstance('i-unknown', { runnerId: 102, runnerName: 'runner-i-unknown' }); + ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [idle, busy, unknown] }] }); + ec2Mock.on(TerminateInstancesCommand).resolves({}); + + const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' as const }); + const result = await createTestProvider().reconcile( + createRequest({ + recoveryOnly: true, + runnerInventoryComplete: true, + removeRunner, + runnerStates: [ + githubState(100, 'runner-i-idle', { status: 'offline', busy: false }), + githubState(101, 'runner-i-busy', { status: 'online', busy: true }), + ], + }), + ); + + expect(result).toMatchObject({ + status: 'retained', + currentRunners: 2, + actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 1 }, + errors: [], + }); + expect(removeRunner).toHaveBeenCalledTimes(1); + expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-idle'] }); + expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-busy', 'i-unknown'] }); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); + }); + it('propagates cancellation instead of converting shutdown into a retry result', async () => { const abort = new AbortController(); abort.abort(new Error('service stopping')); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts index fd0cb54b99..0ace6f76a6 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts @@ -94,7 +94,13 @@ export function createEc2ScaleSetProvider( const state = emptyState(ownedRunners.length); const servingRunners = servingCapacity(normalizedInput, ownedRunners, request, state, now()); - if (servingRunners.length < request.desiredRunners) { + if (request.recoveryOnly) { + if (!request.runnerInventoryComplete) { + state.needsRunnerInventory = true; + } else if (servingRunners.length > 0) { + await scaleDown(normalizedInput, servingRunners, servingRunners.length, request, state, runnerOperations); + } + } else if (servingRunners.length < request.desiredRunners) { const capacityDeficit = request.desiredRunners - servingRunners.length; const availableReplacementSlots = Math.max( 0, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts index b94b09f455..5dd7d7ff98 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts @@ -77,7 +77,7 @@ export async function scaleDown( if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; } else if (isBusyState(githubState)) { state.actions.retainedBusy++; - } else if (isSafeScaleDownState(githubState)) { + } else if (isSafeScaleDownState(githubState, request.recoveryOnly)) { candidates.push({ runner, githubState }); } else { retainUnknown(state, runner.instanceId); diff --git a/lambdas/libs/compute-providers/scale-set.ts b/lambdas/libs/compute-providers/scale-set.ts index 2575f95384..d4d0267dce 100644 --- a/lambdas/libs/compute-providers/scale-set.ts +++ b/lambdas/libs/compute-providers/scale-set.ts @@ -52,6 +52,8 @@ export type RemoveScaleSetRunner = (input: RemoveScaleSetRunnerInput) => Promise export interface ScaleSetReconcileRequest { desiredRunners: number; + /** Recovery-only mode never launches capacity; it only removes confirmed idle runners. */ + recoveryOnly?: boolean; /** Orchestration-owned handoff window before exact runner inventory is required. */ bootTimeoutMinutes: number; /** True only when runnerStates contains the controller's complete, freshly joined Actions and GitHub inventory. */ diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md index 48b282bf0e..43758bb376 100644 --- a/lambdas/services/scale-set/README.md +++ b/lambdas/services/scale-set/README.md @@ -71,14 +71,24 @@ GitHub App ID and private-key values are reloaded from SSM whenever an installat Runtime settings: -| Environment variable | Default | -| --------------------------------------------- | ------- | -| `SCALE_SET_HEALTH_PORT` | `8080` | -| `SCALE_SET_HEALTH_STALE_AFTER_SECONDS` | `180` | -| `SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS` | `110` | -| `SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS` | `10` | -| `SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS` | `1` | -| `SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS` | `30` | +| Environment variable | Default | +| --------------------------------------------- | ------------ | +| `SCALE_SET_HEALTH_PORT` | `8080` | +| `SCALE_SET_HEALTH_STALE_AFTER_SECONDS` | `180` | +| `SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS` | `110` | +| `SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS` | `10` | +| `SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS` | `1` | +| `SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS` | `30` | +| `SCALE_SET_CONTROLLER_MODE` | `controller` | +| `SCALE_SET_JANITOR_INTERVAL_SECONDS` | `300` | + +Set `SCALE_SET_CONTROLLER_MODE=janitor` in a separate container or task. Janitor mode does not open a message +session and does not create a missing scale set. Each pass lists EC2 instances through the exact ownership tags, +joins the Actions-service runner identity with the public GitHub runner status, removes only an exact runner whose +fresh status is explicitly not busy, and terminates the matching EC2 instance only after GitHub removal succeeds. Busy, +unknown-status, mismatched, missing, and otherwise unknown runners are retained. Run this +process independently from the controller so it can clean idle capacity after a controller failure; do not run both +modes against the same scale set unless the janitor is deliberately being used as the recovery owner. ## Reconciliation and health @@ -90,6 +100,11 @@ Messages follow the upstream scale-set listener order: acknowledge first, then a The EC2 provider counts a `config-published` instance as serving only during the orchestration request's boot window (`bootTimeoutMinutes`, default `10`) or after an exact online or `JobStarted` identity is observed. After the window, offline or unknown capacity is retained rather than terminated, and the complete inventory pass allows it to stop suppressing a replacement. Instances left in an earlier or unknown publication state are also retained for operator recovery and never terminated speculatively. EC2 ownership includes a SHA-256 hash of the canonical GitHub configuration scope, preventing the same runner-config name and numeric scale-set ID in another GitHub scope from colliding. A bounded one-instance physical surge may replace ambiguous capacity; once that ceiling is reached, the provider reports retained capacity instead of creating an unbounded replacement loop. +The recovery janitor intentionally has a stricter destructive boundary than normal reconciliation: it only acts on +an exact EC2 ownership match plus an exact Actions runner ID/name match plus a fresh GitHub `busy: false` response. +It treats an unavailable GitHub inventory as unknown and leaves the instance untouched. The GitHub App therefore +needs organization `Self-hosted runners: Read & write` permission for the final status lookup and removal. + - `GET /healthz` reports controller liveness and is used by Docker/ECS. External GitHub outages remain live but degraded to avoid restart loops. - `GET /readyz` reports readiness and returns 503 unless every reconciler is ready. @@ -101,6 +116,23 @@ Build from the repository root: docker build --target runtime -f lambdas/services/scale-set/Dockerfile -t scale-set-controller . ``` +For local recovery, start the janitor as a separate container. The profile file must be mounted inside the container +and the profile's IAM identity must be allowed to describe and terminate only the tagged runner instances, plus read +the configured SSM parameters: + +```shell +docker run --rm --name scale-set-janitor --no-healthcheck \ + -e SCALE_SET_CONTROLLER_MODE=janitor \ + -e SCALE_SET_JANITOR_INTERVAL_SECONDS=300 \ + -e AWS_REGION=eu-west-1 \ + -e AWS_DEFAULT_REGION=eu-west-1 \ + -e AWS_PROFILE=forge-ops-dev \ + -e AWS_SDK_LOAD_CONFIG=1 \ + -v "$HOME/.aws:/home/node/.aws:ro" \ + -e SCALE_SET_CONTROLLER_MANIFEST="$( { sessionCloseTimeoutMs: 10000, reconnectInitialBackoffMs: 1000, reconnectMaxBackoffMs: 30000, + mode: 'controller', + janitorIntervalMs: 300000, }); }); @@ -60,6 +62,19 @@ describe('scale-set service configuration', () => { ).toThrow('must not exceed'); }); + it('supports an independent janitor mode with a bounded poll interval', () => { + expect( + parseScaleSetServiceConfig({ + SCALE_SET_CONTROLLER_MANIFEST: '{}', + SCALE_SET_CONTROLLER_MODE: 'janitor', + SCALE_SET_JANITOR_INTERVAL_SECONDS: '60', + }), + ).toMatchObject({ mode: 'janitor', janitorIntervalMs: 60000 }); + expect(() => + parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: '{}', SCALE_SET_CONTROLLER_MODE: 'invalid' }), + ).toThrow('must be controller or janitor'); + }); + it('validates production selectors and numeric runtime settings', () => { expect(() => parseScaleSetServiceConfig({})).toThrow('provide exactly one'); expect(() => @@ -86,7 +101,7 @@ describe('parseScaleSetControllerManifest', () => { expect(parseScaleSetReconcilerConfig(runnerConfig(), 0, 'group')).toMatchObject({ schemaVersion: 1, runnerConfigName: 'linux-x64', - expectedScaleSetName: 'linux-x64', + scaleSetName: 'linux-x64', bootTimeoutMinutes: 10, sessionOwner: 'group.linux-x64', workFolder: '_work', @@ -187,7 +202,7 @@ describe('parseScaleSetControllerManifest', () => { runnerConfig({ runnerConfigName: 'other', githubConfigUrl: 'https://GITHUB.com/example/' }), ], }), - ).toThrow('scale set ID'); + ).toThrow('duplicated'); }); it('allows the same numeric scale-set ID in different GitHub scopes', () => { diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts index 0f048dbe6c..d7377210fc 100644 --- a/lambdas/services/scale-set/src/config.ts +++ b/lambdas/services/scale-set/src/config.ts @@ -54,6 +54,8 @@ export interface ScaleSetServiceConfig { sessionCloseTimeoutMs: number; reconnectInitialBackoffMs: number; reconnectMaxBackoffMs: number; + mode: 'controller' | 'janitor'; + janitorIntervalMs: number; } export type ScaleSetServiceEnvironment = Readonly>; @@ -67,6 +69,7 @@ const DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 110; const DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS = 10; const DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS = 1; const DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS = 30; +const DEFAULT_JANITOR_INTERVAL_SECONDS = 300; const MAX_RECONCILERS = 1000; const MAX_PROVIDER_CONFIG_NODES = 10_000; const MAX_PROVIDER_CONFIG_DEPTH = 32; @@ -137,6 +140,11 @@ export function parseScaleSetServiceConfig(environment: ScaleSetServiceEnvironme ); } + const mode = environment.SCALE_SET_CONTROLLER_MODE?.trim() || 'controller'; + if (mode !== 'controller' && mode !== 'janitor') { + throw new ScaleSetConfigurationError('SCALE_SET_CONTROLLER_MODE must be controller or janitor'); + } + return { ...(manifest ? { manifest } : {}), ...(groupConfigPath ? { groupConfigPath, groupName, groupRevision } : {}), @@ -165,6 +173,13 @@ export function parseScaleSetServiceConfig(environment: ScaleSetServiceEnvironme }) * 1000, reconnectInitialBackoffMs, reconnectMaxBackoffMs, + mode, + janitorIntervalMs: + parseInteger(environment, 'SCALE_SET_JANITOR_INTERVAL_SECONDS', { + defaultValue: DEFAULT_JANITOR_INTERVAL_SECONDS, + minimum: 10, + maximum: 86400, + }) * 1000, }; } diff --git a/lambdas/services/scale-set/src/controller.ts b/lambdas/services/scale-set/src/controller.ts index f575f60dd2..6d67cb2a3b 100644 --- a/lambdas/services/scale-set/src/controller.ts +++ b/lambdas/services/scale-set/src/controller.ts @@ -39,6 +39,14 @@ export class ScaleSetController { this.health.markStopping(); await Promise.all(completions); } + + async recover(signal: AbortSignal): Promise { + await Promise.all( + this.manifest.reconcilers.map(async (config) => { + await new ScaleSetReconciler(config, this.serviceConfig, this.dependencies).recover(signal); + }), + ); + } } async function waitForAbort(signal: AbortSignal): Promise { diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts index d7672224a6..da56014130 100644 --- a/lambdas/services/scale-set/src/main.ts +++ b/lambdas/services/scale-set/src/main.ts @@ -57,6 +57,13 @@ async function main(): Promise { runnerInventory: new TtlScaleSetRunnerInventoryCache(), }; const controller = new ScaleSetController(manifest, serviceConfig, dependencies, logger); + + if (serviceConfig.mode === 'janitor') { + await runJanitor(controller, manifest.groupName, manifest.revision, serviceConfig.janitorIntervalMs); + await githubHttp.close(); + return; + } + const runtime = new ScaleSetServiceRuntime(serviceConfig, controller); let healthServer: ScaleSetHealthServer | undefined; @@ -90,6 +97,35 @@ async function main(): Promise { } } +async function runJanitor( + controller: ScaleSetController, + groupName: string, + revision: string | undefined, + intervalMs: number, +): Promise { + const abortController = new AbortController(); + const shutdown = (signal: NodeJS.Signals) => { + logger.info('scale_set_janitor_shutdown_requested', { signal, groupName }); + abortController.abort(new Error(`received ${signal}`)); + }; + const onSigterm = () => shutdown('SIGTERM'); + const onSigint = () => shutdown('SIGINT'); + process.once('SIGTERM', onSigterm); + process.once('SIGINT', onSigint); + + try { + logger.info('scale_set_janitor_started', { groupName, revision, intervalMs }); + while (!abortController.signal.aborted) { + await controller.recover(abortController.signal); + await abortableSleep(intervalMs, abortController.signal); + } + } finally { + process.removeListener('SIGTERM', onSigterm); + process.removeListener('SIGINT', onSigint); + logger.info('scale_set_janitor_stopped', { groupName }); + } +} + void main().catch((error) => { logger.error('scale_set_controller_fatal_failure', { error }); process.exitCode = 1; diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index 7e28a09ee6..c0db5fb853 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -92,7 +92,7 @@ function fixture(options: { reconcile: options.reconcile ?? vi.fn().mockResolvedValue(result()), }; const client: ScaleSetReconcilerClient = { - getRunnerScaleSetById: vi.fn().mockResolvedValue({ id: 42, name: 'linux' }), + getRunnerScaleSetById: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), getRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), createRunnerScaleSet: vi.fn().mockResolvedValue({ id: 42, name: 'linux', runnerGroupId: 7 }), getRunnerGroupByName: vi.fn().mockResolvedValue({ @@ -135,6 +135,7 @@ describe('ScaleSetReconciler', () => { session: { statistics: undefined }, getMessage: vi.fn().mockResolvedValue(message()), deleteMessage: vi.fn(), + acquireJobs: vi.fn(), close: vi.fn(), }; const { client, dependencies } = fixture({ @@ -178,6 +179,7 @@ describe('ScaleSetReconciler', () => { session: { statistics: undefined }, getMessage: vi.fn().mockResolvedValue(message()), deleteMessage: vi.fn(), + acquireJobs: vi.fn(), close: vi.fn(), }; const { client, dependencies } = fixture({ @@ -206,6 +208,34 @@ describe('ScaleSetReconciler', () => { ); }); + it('runs recovery without opening a session or registering a missing scale set', async () => { + const abort = new AbortController(); + const session = { + session: { statistics: undefined }, + getMessage: vi.fn(), + close: vi.fn(), + }; + const reconcile = vi.fn().mockResolvedValue(result({ desiredRunners: 0, currentRunners: 0 })); + const { client, computeProvider, dependencies } = fixture({ session, reconcile }); + vi.mocked(client.getRunnerScaleSetById).mockResolvedValue({ id: 42, name: 'linux' }); + + await new ScaleSetReconciler(config, serviceConfig, dependencies).recover(abort.signal); + + expect(client.createMessageSessionClient).not.toHaveBeenCalled(); + expect(client.createRunnerScaleSet).not.toHaveBeenCalled(); + expect(computeProvider.reconcile).toHaveBeenCalledWith( + expect.objectContaining({ + desiredRunners: 0, + recoveryOnly: true, + runnerInventoryComplete: true, + }), + ); + expect(dependencies.logger.info).toHaveBeenCalledWith( + 'scale_set_recovery_reconciled', + expect.objectContaining({ actions: expect.any(Object) }), + ); + }); + it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { const order: string[] = []; const abort = new AbortController(); @@ -524,7 +554,9 @@ describe('ScaleSetReconciler', () => { const reconciler = new ScaleSetReconciler(config, serviceConfig, dependencies) as unknown as { rememberLifecycle(id: number, name: string, lifecycle: 'started'): void; lifecycle: Map; + resolvedScaleSetId: number; }; + reconciler.resolvedScaleSetId = 42; for (let index = 0; index < 1100; index += 1) reconciler.rememberLifecycle(index + 1, `runner-${index}`, 'started'); expect(reconciler.lifecycle.size).toBe(1000); expect(reconciler.lifecycle.has('runner-0')).toBe(false); diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index fce47a04f5..e9ec904070 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -134,7 +134,7 @@ export class ScaleSetReconciler { try { const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); client = this.dependencies.createClient(this.config, accessTokenProvider); - const resolved = await this.resolveScaleSet(client, signal); + const resolved = await this.resolveScaleSet(client, signal, true); this.resolvedScaleSetId = resolved.scaleSetId; this.resolvedRunnerGroupId = resolved.runnerGroupId; client.setSystemInfo({ ...client.systemInfo, scaleSetId: resolved.scaleSetId }); @@ -239,9 +239,66 @@ export class ScaleSetReconciler { status.markStopping(); } + /** Run one independent recovery pass without opening a message session or creating a scale set. */ + async recover(signal: AbortSignal): Promise { + let provider: ScaleSetComputeProvider; + let client: ScaleSetReconcilerClient; + try { + const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); + client = this.dependencies.createClient(this.config, accessTokenProvider); + const resolved = await this.resolveScaleSet(client, signal, false); + this.resolvedScaleSetId = resolved.scaleSetId; + this.resolvedRunnerGroupId = resolved.runnerGroupId; + client.setSystemInfo({ ...client.systemInfo, scaleSetId: resolved.scaleSetId }); + provider = this.dependencies.computeProviders.create(this.config.computeProvider.type, { + runnerConfigName: this.config.runnerConfigName, + scaleSetId: resolved.scaleSetId, + githubScope: this.config.githubConfigUrl, + configuration: this.config.computeProvider.configuration, + }); + } catch (error) { + if (signal.aborted) return; + this.log('warn', 'scale_set_recovery_initialization_failed', { + computeProviderType: this.config.computeProvider.type, + ...httpErrorLogAttributes(error), + error, + }); + return; + } + + try { + const inventory = await this.loadScaleSetInventory(client, signal); + const result = await this.reconcileProvider(provider, { + desiredRunners: 0, + recoveryOnly: true, + bootTimeoutMinutes: this.config.bootTimeoutMinutes, + runnerInventoryComplete: true, + runnerStates: this.mergeLifecycle(inventory), + ...this.createReconcileCallbacks(client, signal), + }); + validateProviderResult(result, 0); + throwIfProviderError(result); + this.log('info', 'scale_set_recovery_reconciled', { + computeProviderType: this.config.computeProvider.type, + currentRunners: result.currentRunners, + status: result.status, + actions: result.actions, + errorCount: result.errors.length, + }); + } catch (error) { + if (signal.aborted) return; + this.log('warn', 'scale_set_recovery_failed', { + computeProviderType: this.config.computeProvider.type, + ...httpErrorLogAttributes(error), + error, + }); + } + } + private async resolveScaleSet( client: ScaleSetReconcilerClient, signal: AbortSignal, + registerMissing: boolean, ): Promise<{ scaleSetId: number; runnerGroupId?: number }> { let runnerGroupId = this.config.expectedRunnerGroupId; if (this.config.runnerGroupName !== undefined) { @@ -273,7 +330,12 @@ export class ScaleSetReconciler { this.config.scaleSetId === undefined ? await client.getRunnerScaleSet(runnerGroupId as number, this.config.scaleSetName, { signal }) : await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); - if (configuredScaleSet === null && runnerGroupId !== undefined && this.config.scaleSetId === undefined) { + if ( + registerMissing && + configuredScaleSet === null && + runnerGroupId !== undefined && + this.config.scaleSetId === undefined + ) { this.log('info', 'scale_set_registering', { runnerConfigName: this.config.runnerConfigName, scaleSetName: this.config.scaleSetName, @@ -354,7 +416,49 @@ export class ScaleSetReconciler { this.config.minRunners, this.config.maxRunners, ); - const callbacks = { + const callbacks = this.createReconcileCallbacks(client, signal); + let result = await this.reconcileProvider(provider, { + desiredRunners, + bootTimeoutMinutes: this.config.bootTimeoutMinutes, + runnerInventoryComplete: false, + runnerStates: this.lifecycleStates(), + ...callbacks, + }); + validateProviderResult(result, desiredRunners); + throwIfProviderError(result); + if (result.needsRunnerInventory) { + let inventory: readonly GitHubScaleSetRunnerState[]; + try { + inventory = await this.loadScaleSetInventory(client, signal); + } catch (error) { + if (!isScaleSetHttpError(error) || error.status !== 404) throw error; + this.log('warn', 'scale_set_runner_inventory_unavailable', { + ...httpErrorLogAttributes(error), + error, + }); + this.logReconciliationResult(result, desiredRunners); + return; + } + result = await this.reconcileProvider(provider, { + desiredRunners, + bootTimeoutMinutes: this.config.bootTimeoutMinutes, + runnerInventoryComplete: true, + runnerStates: this.mergeLifecycle(inventory), + ...callbacks, + }); + validateProviderResult(result, desiredRunners); + throwIfProviderError(result); + if (result.needsRunnerInventory) { + throw new ScaleSetProtocolError( + 'scale-set compute provider requested inventory after a complete inventory pass', + ); + } + } + this.logReconciliationResult(result, desiredRunners); + } + + private createReconcileCallbacks(client: ScaleSetReconcilerClient, signal: AbortSignal) { + return { signal, generateJitConfiguration: async ({ runnerName, @@ -429,44 +533,6 @@ export class ScaleSetReconciler { return { status: 'removed' as const }; }, }; - let result = await this.reconcileProvider(provider, { - desiredRunners, - bootTimeoutMinutes: this.config.bootTimeoutMinutes, - runnerInventoryComplete: false, - runnerStates: this.lifecycleStates(), - ...callbacks, - }); - validateProviderResult(result, desiredRunners); - throwIfProviderError(result); - if (result.needsRunnerInventory) { - let inventory: readonly GitHubScaleSetRunnerState[]; - try { - inventory = await this.loadScaleSetInventory(client, signal); - } catch (error) { - if (!isScaleSetHttpError(error) || error.status !== 404) throw error; - this.log('warn', 'scale_set_runner_inventory_unavailable', { - ...httpErrorLogAttributes(error), - error, - }); - this.logReconciliationResult(result, desiredRunners); - return; - } - result = await this.reconcileProvider(provider, { - desiredRunners, - bootTimeoutMinutes: this.config.bootTimeoutMinutes, - runnerInventoryComplete: true, - runnerStates: this.mergeLifecycle(inventory), - ...callbacks, - }); - validateProviderResult(result, desiredRunners); - throwIfProviderError(result); - if (result.needsRunnerInventory) { - throw new ScaleSetProtocolError( - 'scale-set compute provider requested inventory after a complete inventory pass', - ); - } - } - this.logReconciliationResult(result, desiredRunners); } private logReconciliationResult(result: ScaleSetReconcileResult, desiredRunners: number): void { @@ -496,6 +562,7 @@ export class ScaleSetReconciler { this.log('info', 'scale_set_compute_provider_reconcile_started', { computeProviderType: this.config.computeProvider.type, desiredRunners: request.desiredRunners, + recoveryOnly: request.recoveryOnly ?? false, runnerInventoryComplete: request.runnerInventoryComplete, }); try { @@ -505,6 +572,7 @@ export class ScaleSetReconciler { this.log('error', 'scale_set_compute_provider_reconcile_failed', { computeProviderType: this.config.computeProvider.type, desiredRunners: request.desiredRunners, + recoveryOnly: request.recoveryOnly ?? false, runnerInventoryComplete: request.runnerInventoryComplete, error, }); From 91f880e0f530b8aeb851b7249058ad28aa6e3267 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 15:51:38 +0200 Subject: [PATCH 18/19] fix(scale-set): add runner inventory diagnostics --- lambdas/services/scale-set/src/reconciler.ts | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index e9ec904070..ff46d9806a 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -267,7 +267,16 @@ export class ScaleSetReconciler { } try { + this.log('info', 'scale_set_runner_inventory_loading', { + githubConfigUrl: this.config.githubConfigUrl, + githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', + inventorySources: ['actions_service', 'github_rest'], + }); const inventory = await this.loadScaleSetInventory(client, signal); + this.log('info', 'scale_set_runner_inventory_loaded', { + githubConfigUrl: this.config.githubConfigUrl, + runnerCount: inventory.length, + }); const result = await this.reconcileProvider(provider, { desiredRunners: 0, recoveryOnly: true, @@ -288,6 +297,13 @@ export class ScaleSetReconciler { } catch (error) { if (signal.aborted) return; this.log('warn', 'scale_set_recovery_failed', { + failureStage: 'github_runner_inventory_or_ec2_recovery', + githubConfigUrl: this.config.githubConfigUrl, + githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', + diagnosis: + isScaleSetHttpError(error) && error.status === 404 + ? 'github_endpoint_not_found_or_app_not_authorized' + : undefined, computeProviderType: this.config.computeProvider.type, ...httpErrorLogAttributes(error), error, @@ -429,10 +445,23 @@ export class ScaleSetReconciler { if (result.needsRunnerInventory) { let inventory: readonly GitHubScaleSetRunnerState[]; try { + this.log('info', 'scale_set_runner_inventory_loading', { + githubConfigUrl: this.config.githubConfigUrl, + githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', + inventorySources: ['actions_service', 'github_rest'], + }); inventory = await this.loadScaleSetInventory(client, signal); + this.log('info', 'scale_set_runner_inventory_loaded', { + githubConfigUrl: this.config.githubConfigUrl, + runnerCount: inventory.length, + }); } catch (error) { if (!isScaleSetHttpError(error) || error.status !== 404) throw error; this.log('warn', 'scale_set_runner_inventory_unavailable', { + failureStage: 'github_runner_inventory', + githubConfigUrl: this.config.githubConfigUrl, + githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', + diagnosis: 'github_endpoint_not_found_or_app_not_authorized', ...httpErrorLogAttributes(error), error, }); From fc511662795b8e88ba33adf85a5c7500b321eb82 Mon Sep 17 00:00:00 2001 From: edersonbrilhante Date: Fri, 28 Aug 2026 17:11:42 +0200 Subject: [PATCH 19/19] fix(scale-set): align provider reconciliation with Actions service --- .../aws/ec2/src/scale-set/inventory.test.ts | 31 +- .../aws/ec2/src/scale-set/inventory.ts | 43 +-- .../aws/ec2/src/scale-set/provider.test.ts | 51 +-- .../aws/ec2/src/scale-set/provider.ts | 14 +- .../aws/ec2/src/scale-set/reconcile.ts | 19 +- .../aws/ec2/src/scale-set/scale-down.test.ts | 46 +-- .../aws/ec2/src/scale-set/scale-down.ts | 27 +- .../aws/ec2/src/scale-set/scale-up.test.ts | 1 - .../aws/ec2/src/scale-set/test/fixtures.ts | 2 +- lambdas/libs/compute-providers/scale-set.ts | 9 +- .../src/client.test.ts | 17 - .../github-actions-scale-set/src/client.ts | 114 ------- .../github-actions-scale-set/src/types.ts | 15 - lambdas/services/scale-set/README.md | 55 +--- lambdas/services/scale-set/src/config.test.ts | 15 - lambdas/services/scale-set/src/config.ts | 15 - lambdas/services/scale-set/src/controller.ts | 12 +- lambdas/services/scale-set/src/logger.test.ts | 16 +- lambdas/services/scale-set/src/logger.ts | 37 ++- lambdas/services/scale-set/src/main.ts | 44 +-- .../services/scale-set/src/reconciler.test.ts | 206 +----------- lambdas/services/scale-set/src/reconciler.ts | 298 ++---------------- 22 files changed, 179 insertions(+), 908 deletions(-) diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts index fc03dc3612..c4b0347376 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.test.ts @@ -61,7 +61,6 @@ describe('EC2 scale-set inventory', () => { expect(result).toMatchObject({ status: 'converged', currentRunners: 1, - needsRunnerInventory: false, actions: { launched: 0, retainedUnknown: 0 }, }); expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); @@ -81,40 +80,23 @@ describe('EC2 scale-set inventory', () => { ); expect(result).toMatchObject({ - status: 'retained', + status: 'converged', currentRunners: 1, - needsRunnerInventory: true, - actions: { launched: 0, retainedUnknown: 1 }, + actions: { launched: 0, retainedUnknown: 0 }, }); expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); }); - it('requests a complete inventory for an old handoff, then counts only its exact online identity', async () => { + it('counts tagged capacity after the boot window without public runner inventory', async () => { const instance = ownedInstance('i-old', { runnerId: 101, runnerName: 'runner-i-old' }); ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); - const computeProvider = createTestProvider({ now: () => new Date('2026-08-24T10:10:00Z').getTime() }); - - const firstPass = await computeProvider.reconcile(createRequest()); - - expect(firstPass).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: true, - actions: { launched: 0, retainedUnknown: 1 }, - }); - expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); - - const secondPass = await computeProvider.reconcile( - createRequest({ - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-i-old', { status: 'online', lifecycle: 'unknown' })], - }), + const result = await createTestProvider({ now: () => new Date('2026-08-24T10:10:00Z').getTime() }).reconcile( + createRequest({ busyRunners: 1 }), ); - expect(secondPass).toMatchObject({ + expect(result).toMatchObject({ status: 'converged', currentRunners: 1, - needsRunnerInventory: false, actions: { launched: 0, retainedUnknown: 0 }, }); expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); @@ -135,7 +117,6 @@ describe('EC2 scale-set inventory', () => { expect(result).toMatchObject({ status: 'converged', currentRunners: 1, - needsRunnerInventory: false, actions: { launched: 0, retainedUnknown: 0 }, }); }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts index aac37bd4c8..91941b0139 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/inventory.ts @@ -206,23 +206,6 @@ export function servingCapacity( const serving: OwnedEc2Runner[] = []; for (const runner of runners) { - const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); - if (request.recoveryOnly) { - if ( - githubState !== undefined && - (githubState.status === 'online' || githubState.status === 'offline') && - typeof githubState.busy === 'boolean' - ) { - // Recovery gets a fresh public GitHub status and may classify an - // offline, non-busy runner as removable. Unknown and busy identities - // remain in the scale-down classifier and are retained there. - serving.push(runner); - } else { - retainUnknown(state, runner.instanceId); - } - continue; - } - if (runner.scaleSetState !== 'config-published') { // An interrupted publication may already have been consumed. Preserve it, // but do not let it suppress replacement capacity indefinitely. @@ -230,6 +213,7 @@ export function servingCapacity( continue; } + const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); if (githubState !== undefined && isConfirmedServingState(githubState)) { serving.push(runner); continue; @@ -239,26 +223,25 @@ export function servingCapacity( continue; } - retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) { - // Treat the stale handoff provisionally as serving until the controller - // supplies one complete joined inventory. This avoids a blind replacement - // before GitHub identity can be checked. - state.needsRunnerInventory = true; + // A config-published EC2 instance is provider-owned capacity. The exact + // Actions-service identity is used only when removing it; no public + // GitHub runner inventory is needed to count capacity. + if (runner.githubRunnerId !== undefined && runner.runnerName !== undefined) { serving.push(runner); + continue; } + + retainUnknown(state, runner.instanceId); + // Unknown lifecycle state is not allowed to suppress replacement capacity. + // Aggregate busy state still protects scale-down, while tagged EC2 + // inventory remains the source of current provider-owned capacity. } return serving; } -export function isSafeScaleDownState(state: ScaleSetRunnerState, recoveryOnly = false): boolean { - return ( - (state.lifecycle === 'completed' && state.busy !== true) || - (state.lifecycle !== 'started' && - (state.status === 'online' || (recoveryOnly && state.status === 'offline')) && - state.busy === false) - ); +export function isSafeScaleDownState(state: ScaleSetRunnerState): boolean { + return state.busy === false || (state.lifecycle === 'completed' && state.busy !== true); } export function isBusyState(state: ScaleSetRunnerState): boolean { diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts index 5b17142e12..704283ac64 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.test.ts @@ -14,7 +14,7 @@ describe('EC2 scale-set provider orchestration', () => { ); }); - it('retains an old offline handoff and bounds replacement to one physical surge instance', async () => { + it('counts an old tagged handoff as provider capacity without public inventory', async () => { const old = ownedInstance('i-old-offline', { runnerId: 100, runnerName: 'runner-i-old-offline' }); const replacementId = 'i-1234567890abcdef0'; const replacement = ownedInstance( @@ -31,7 +31,6 @@ describe('EC2 scale-set provider orchestration', () => { ec2Mock.on(CreateFleetCommand).resolves({ Instances: [{ InstanceIds: [replacementId] }] }); const computeProvider = createTestProvider({ now: () => new Date('2026-08-24T10:11:00Z').getTime() }); const completeInventory = createRequest({ - runnerInventoryComplete: true, runnerStates: [ githubState(100, 'runner-i-old-offline', { status: 'offline', @@ -45,18 +44,16 @@ describe('EC2 scale-set provider orchestration', () => { const nextResult = await computeProvider.reconcile(completeInventory); expect(result).toMatchObject({ - status: 'retained', - currentRunners: 2, - needsRunnerInventory: false, - actions: { launched: 1, retainedUnknown: 1 }, + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, }); expect(nextResult).toMatchObject({ - status: 'retained', - currentRunners: 2, - needsRunnerInventory: false, - actions: { launched: 0, retainedUnknown: 1 }, + status: 'converged', + currentRunners: 1, + actions: { launched: 0, retainedUnknown: 0 }, }); - expect(ec2Mock).toHaveReceivedCommandTimes(CreateFleetCommand, 1); + expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); }); it.each(['provisioning', 'publishing'])( @@ -123,38 +120,6 @@ describe('EC2 scale-set provider orchestration', () => { expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); }); - it('recovery removes only exact idle runners and retains busy or unknown runners', async () => { - const idle = ownedInstance('i-idle', { runnerId: 100, runnerName: 'runner-i-idle' }); - const busy = ownedInstance('i-busy', { runnerId: 101, runnerName: 'runner-i-busy' }); - const unknown = ownedInstance('i-unknown', { runnerId: 102, runnerName: 'runner-i-unknown' }); - ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [idle, busy, unknown] }] }); - ec2Mock.on(TerminateInstancesCommand).resolves({}); - - const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' as const }); - const result = await createTestProvider().reconcile( - createRequest({ - recoveryOnly: true, - runnerInventoryComplete: true, - removeRunner, - runnerStates: [ - githubState(100, 'runner-i-idle', { status: 'offline', busy: false }), - githubState(101, 'runner-i-busy', { status: 'online', busy: true }), - ], - }), - ); - - expect(result).toMatchObject({ - status: 'retained', - currentRunners: 2, - actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 1 }, - errors: [], - }); - expect(removeRunner).toHaveBeenCalledTimes(1); - expect(ec2Mock).toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-idle'] }); - expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-busy', 'i-unknown'] }); - expect(ec2Mock).not.toHaveReceivedCommand(CreateFleetCommand); - }); - it('propagates cancellation instead of converting shutdown into a retry result', async () => { const abort = new AbortController(); abort.abort(new Error('service stopping')); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts index 0ace6f76a6..3c9df100ed 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/provider.ts @@ -16,8 +16,8 @@ import { safeError, throwIfAborted, validateBootTimeout, + validateBusyRunners, validateDesiredRunners, - validateInventorySignal, } from './reconcile'; import { scaleDown } from './scale-down'; import { scaleUp } from './scale-up'; @@ -72,8 +72,8 @@ export function createEc2ScaleSetProvider( request.signal.throwIfAborted(); const validationError = validateDesiredRunners(request.desiredRunners) ?? - validateBootTimeout(request.bootTimeoutMinutes) ?? - validateInventorySignal(request.runnerInventoryComplete); + validateBusyRunners(request.busyRunners) ?? + validateBootTimeout(request.bootTimeoutMinutes); if (validationError) { const state = emptyState(0); state.errors.push(validationError); @@ -94,13 +94,7 @@ export function createEc2ScaleSetProvider( const state = emptyState(ownedRunners.length); const servingRunners = servingCapacity(normalizedInput, ownedRunners, request, state, now()); - if (request.recoveryOnly) { - if (!request.runnerInventoryComplete) { - state.needsRunnerInventory = true; - } else if (servingRunners.length > 0) { - await scaleDown(normalizedInput, servingRunners, servingRunners.length, request, state, runnerOperations); - } - } else if (servingRunners.length < request.desiredRunners) { + if (servingRunners.length < request.desiredRunners) { const capacityDeficit = request.desiredRunners - servingRunners.length; const availableReplacementSlots = Math.max( 0, diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts index d4341abe1e..a49e8c0293 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/reconcile.ts @@ -9,7 +9,6 @@ const MAX_BOOT_TIMEOUT_MINUTES = 120; export interface MutableReconcileState { currentRunners: number; - needsRunnerInventory: boolean; retainedUnknownResourceIds: Set; actions: ScaleSetReconcileActions; errors: ScaleSetReconcileError[]; @@ -56,14 +55,8 @@ export function throwIfAborted(signal: AbortSignal, error?: unknown): void { } } -function resultStatus( - errors: readonly ScaleSetReconcileError[], - current: number, - desired: number, - needsRunnerInventory: boolean, -) { +function resultStatus(errors: readonly ScaleSetReconcileError[], current: number, desired: number) { if (errors.length > 0 || current < desired) return 'error' as const; - if (needsRunnerInventory) return 'retained' as const; if (current > desired) return 'retained' as const; return 'converged' as const; } @@ -76,10 +69,9 @@ export function finish(state: MutableReconcileState, desiredRunners: number): Sc }); } return { - status: resultStatus(state.errors, state.currentRunners, desiredRunners, state.needsRunnerInventory), + status: resultStatus(state.errors, state.currentRunners, desiredRunners), desiredRunners, currentRunners: state.currentRunners, - needsRunnerInventory: state.needsRunnerInventory, actions: state.actions, errors: state.errors, }; @@ -88,7 +80,6 @@ export function finish(state: MutableReconcileState, desiredRunners: number): Sc export function emptyState(currentRunners: number): MutableReconcileState { return { currentRunners, - needsRunnerInventory: false, retainedUnknownResourceIds: new Set(), actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, errors: [], @@ -129,11 +120,11 @@ export function validateBootTimeout(bootTimeoutMinutes: number): ScaleSetReconci return undefined; } -export function validateInventorySignal(runnerInventoryComplete: unknown): ScaleSetReconcileError | undefined { - if (typeof runnerInventoryComplete !== 'boolean') { +export function validateBusyRunners(busyRunners: number): ScaleSetReconcileError | undefined { + if (!Number.isSafeInteger(busyRunners) || busyRunners < 0 || busyRunners > 2_147_483_647) { return { operation: 'validate', - code: 'INVALID_RUNNER_INVENTORY_SIGNAL', + code: 'INVALID_BUSY_RUNNER_COUNT', }; } return undefined; diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts index 1b3b5e0c60..40b64e1c2b 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.test.ts @@ -17,7 +17,6 @@ describe('EC2 scale-set scale down', () => { const result = await createTestProvider().reconcile( createRequest({ desiredRunners: 2, - runnerInventoryComplete: true, runnerStates: [ githubState(101, 'runner-completed', { status: 'offline', busy: undefined, lifecycle: 'completed' }), githubState(102, 'runner-busy', { busy: true, lifecycle: 'started' }), @@ -30,8 +29,7 @@ describe('EC2 scale-set scale down', () => { status: 'converged', desiredRunners: 2, currentRunners: 2, - needsRunnerInventory: false, - actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 1 }, + actions: { launched: 0, terminated: 1, retainedBusy: 1, retainedUnknown: 0 }, errors: [], }); expect(removeRunner).toHaveBeenCalledTimes(1); @@ -46,38 +44,17 @@ describe('EC2 scale-set scale down', () => { expect(ec2Mock).not.toHaveReceivedCommandWith(TerminateInstancesCommand, { InstanceIds: ['i-unknown'] }); }); - it('uses a typed inventory signal for a conservative first pass and exact second pass', async () => { + it('uses aggregate idle state to remove a tagged runner after a restart', async () => { const instance = ownedInstance('i-completed', { runnerId: 101, runnerName: 'runner-completed' }); ec2Mock.on(DescribeInstancesCommand).resolves({ Reservations: [{ Instances: [instance] }] }); const removeRunner = vi.fn().mockResolvedValue({ status: 'removed' }); - const computeProvider = createTestProvider(); - - const firstPass = await computeProvider.reconcile( - createRequest({ desiredRunners: 0, runnerStates: [], removeRunner }), - ); - - expect(firstPass).toMatchObject({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: true, - actions: { terminated: 0, retainedUnknown: 1 }, - errors: [], - }); - expect(removeRunner).not.toHaveBeenCalled(); - - const secondPass = await computeProvider.reconcile( - createRequest({ - desiredRunners: 0, - runnerInventoryComplete: true, - runnerStates: [githubState(101, 'runner-completed', { lifecycle: 'completed', status: 'offline' })], - removeRunner, - }), + const result = await createTestProvider().reconcile( + createRequest({ desiredRunners: 0, busyRunners: 0, runnerStates: [], removeRunner }), ); - expect(secondPass).toMatchObject({ + expect(result).toMatchObject({ status: 'converged', currentRunners: 0, - needsRunnerInventory: false, actions: { terminated: 1 }, }); expect(removeRunner).toHaveBeenCalledTimes(1); @@ -91,7 +68,7 @@ describe('EC2 scale-set scale down', () => { const result = await createTestProvider().reconcile( createRequest({ desiredRunners: 0, - runnerInventoryComplete: true, + busyRunners: 1, runnerStates: [ githubState(101, 'runner-completed-busy', { lifecycle: 'completed', @@ -106,7 +83,6 @@ describe('EC2 scale-set scale down', () => { expect(result).toMatchObject({ status: 'retained', currentRunners: 1, - needsRunnerInventory: false, actions: { terminated: 0, retainedBusy: 1 }, errors: [], }); @@ -122,7 +98,7 @@ describe('EC2 scale-set scale down', () => { const result = await createTestProvider().reconcile( createRequest({ desiredRunners: 0, - runnerInventoryComplete: true, + busyRunners: 0, runnerStates: [githubState(101, 'runner-raced-busy')], removeRunner, }), @@ -131,7 +107,6 @@ describe('EC2 scale-set scale down', () => { expect(result).toMatchObject({ status: 'retained', currentRunners: 1, - needsRunnerInventory: false, actions: { terminated: 0, retainedBusy: 1, retainedUnknown: 0 }, errors: [], }); @@ -146,7 +121,7 @@ describe('EC2 scale-set scale down', () => { const result = await createTestProvider().reconcile( createRequest({ desiredRunners: 0, - runnerInventoryComplete: true, + busyRunners: 0, runnerStates: [githubState(101, 'runner-raced-unknown')], removeRunner, }), @@ -155,7 +130,6 @@ describe('EC2 scale-set scale down', () => { expect(result).toMatchObject({ status: 'retained', currentRunners: 1, - needsRunnerInventory: false, actions: { terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, errors: [], }); @@ -170,7 +144,7 @@ describe('EC2 scale-set scale down', () => { const result = await createTestProvider().reconcile( createRequest({ desiredRunners: 0, - runnerInventoryComplete: true, + busyRunners: 0, runnerStates: [githubState(101, 'runner-exact')], removeRunner, }), @@ -196,7 +170,7 @@ describe('EC2 scale-set scale down', () => { const result = await createTestProvider().reconcile( createRequest({ desiredRunners: 0, - runnerInventoryComplete: true, + busyRunners: 0, runnerStates: [githubState(101, 'runner-idle')], removeRunner, }), diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts index 5dd7d7ff98..78fddce9cd 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-down.ts @@ -40,7 +40,6 @@ async function terminateKnownIdleRunner( } if (removalResult.status !== 'removed') { retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; return false; } @@ -72,16 +71,34 @@ export async function scaleDown( for (const runner of runners) { const githubState = matchingRunnerState(runner, runnerStateIndex, input.scaleSetId); + const hasContradictoryState = runner.runnerName !== undefined && runnerStateIndex.byName.has(runner.runnerName); if (!githubState) { - retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; + if ( + !hasContradictoryState && + request.busyRunners === 0 && + runner.githubRunnerId !== undefined && + runner.runnerName !== undefined + ) { + candidates.push({ + runner, + githubState: { + runnerId: runner.githubRunnerId, + runnerName: runner.runnerName, + scaleSetId: input.scaleSetId, + status: 'unknown', + busy: false, + lifecycle: 'unknown', + }, + }); + } else { + retainUnknown(state, runner.instanceId); + } } else if (isBusyState(githubState)) { state.actions.retainedBusy++; - } else if (isSafeScaleDownState(githubState, request.recoveryOnly)) { + } else if (isSafeScaleDownState(githubState)) { candidates.push({ runner, githubState }); } else { retainUnknown(state, runner.instanceId); - if (!request.runnerInventoryComplete) state.needsRunnerInventory = true; } } diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts index f0a962faa7..d99d19ebbc 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/scale-up.test.ts @@ -31,7 +31,6 @@ describe('EC2 scale-set scale up', () => { status: 'converged', desiredRunners: 1, currentRunners: 1, - needsRunnerInventory: false, actions: { launched: 1, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, errors: [], }); diff --git a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts index 2f770dfdbf..abbd63a16c 100644 --- a/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts +++ b/lambdas/libs/compute-providers/aws/ec2/src/scale-set/test/fixtures.ts @@ -100,8 +100,8 @@ export function jitResult(instanceId = 'i-1234567890abcdef0'): GenerateScaleSetJ export function createRequest(overrides: Partial = {}): ScaleSetReconcileRequest { return { desiredRunners: 1, + busyRunners: 0, bootTimeoutMinutes: 10, - runnerInventoryComplete: false, runnerStates: [], signal, generateJitConfiguration: vi.fn().mockResolvedValue(jitResult()), diff --git a/lambdas/libs/compute-providers/scale-set.ts b/lambdas/libs/compute-providers/scale-set.ts index d4d0267dce..860138bbe9 100644 --- a/lambdas/libs/compute-providers/scale-set.ts +++ b/lambdas/libs/compute-providers/scale-set.ts @@ -52,12 +52,9 @@ export type RemoveScaleSetRunner = (input: RemoveScaleSetRunnerInput) => Promise export interface ScaleSetReconcileRequest { desiredRunners: number; - /** Recovery-only mode never launches capacity; it only removes confirmed idle runners. */ - recoveryOnly?: boolean; - /** Orchestration-owned handoff window before exact runner inventory is required. */ + /** Aggregate busy-runner count reported by the GitHub Actions scale-set session. */ + busyRunners: number; bootTimeoutMinutes: number; - /** True only when runnerStates contains the controller's complete, freshly joined Actions and GitHub inventory. */ - runnerInventoryComplete: boolean; runnerStates: readonly ScaleSetRunnerState[]; signal: AbortSignal; generateJitConfiguration: GenerateScaleSetJitConfiguration; @@ -96,8 +93,6 @@ export interface ScaleSetReconcileResult { desiredRunners: number; /** Best-known owned capacity after actions completed; the next reconciliation re-observes AWS. */ currentRunners: number; - /** The provider retained unknown capacity and needs a controller inventory refresh before retrying scale-down. */ - needsRunnerInventory: boolean; actions: ScaleSetReconcileActions; errors: readonly ScaleSetReconcileError[]; } diff --git a/lambdas/libs/github-actions-scale-set/src/client.test.ts b/lambdas/libs/github-actions-scale-set/src/client.test.ts index a1b23c8aef..246ee8cbb8 100644 --- a/lambdas/libs/github-actions-scale-set/src/client.test.ts +++ b/lambdas/libs/github-actions-scale-set/src/client.test.ts @@ -153,23 +153,6 @@ describe('GitHubActionsScaleSetClient', () => { await expect(client.getRunnerScaleSetById(42)).rejects.toBeInstanceOf(ScaleSetProtocolError); }); - it('fetches an exact public GitHub runner immediately by id', async () => { - const fixture = clientFixture((url) => { - if (url.pathname === '/orgs/example/actions/runners/71') { - return jsonResponse({ id: 71, name: 'runner-71', status: 'online', busy: false }); - } - return new Response(null, { status: 500 }); - }); - - await expect(fixture.client.getGitHubRunner(71)).resolves.toEqual({ - id: 71, - name: 'runner-71', - status: 'online', - busy: false, - }); - expect(fixture.accessTokenProvider).toHaveBeenCalledOnce(); - }); - it('refreshes the Actions admin token when it enters the 60-second expiry window', async () => { let nowMs = Date.UTC(2026, 7, 14, 12, 0, 0); let tokenIssue = 0; diff --git a/lambdas/libs/github-actions-scale-set/src/client.ts b/lambdas/libs/github-actions-scale-set/src/client.ts index 89b4d96cb2..36e89cd586 100644 --- a/lambdas/libs/github-actions-scale-set/src/client.ts +++ b/lambdas/libs/github-actions-scale-set/src/client.ts @@ -7,13 +7,11 @@ import { AccessTokenProvider, GitHubActionsScaleSetClientOptions, RunnerGroup, - GitHubRunnerReference, RunnerReference, RunnerScaleSet, RunnerScaleSetJitRunnerConfig, RunnerScaleSetJitRunnerSetting, ScaleSetFetch, - ScaleSetRunnerState, ScaleSetRequestOptions, SystemInfo, } from './types'; @@ -53,13 +51,6 @@ interface RunnerReferenceListResponse { value: RunnerReference[]; } -interface GitHubRunnerListResponse { - total_count: number; - runners: GitHubRunnerReference[]; -} - -const MAX_GITHUB_RUNNER_PAGES = 100; - interface ActionsRequestOptions extends ScaleSetRequestOptions { query?: Record; body?: unknown; @@ -372,100 +363,6 @@ export class GitHubActionsScaleSetClient { return parseJsonResponse(result, 'GET', url); } - async listRunners(options: ScaleSetRequestOptions = {}): Promise { - const { result, url } = await this.actionsRequest('GET', RUNNER_ENDPOINT, { - expectedStatuses: [200], - signal: options.signal, - }); - return parseJsonResponse(result, 'GET', url).value; - } - - async listGitHubRunners(options: ScaleSetRequestOptions = {}): Promise { - const runners: GitHubRunnerReference[] = []; - for (let page = 1; page <= MAX_GITHUB_RUNNER_PAGES; page += 1) { - const url = githubApiUrl(this.config, `${this.runnerListPath()}?per_page=100&page=${page}`); - const accessToken = await this.getAccessToken(); - const result = await executeRequest( - this.fetchImplementation, - url, - { - method: 'GET', - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${accessToken}`, - 'User-Agent': this.currentUserAgent, - 'X-GitHub-Api-Version': '2022-11-28', - }, - signal: options.signal, - }, - [200], - ); - const response = parseJsonResponse(result, 'GET', url); - if (!Array.isArray(response.runners)) { - throw new ScaleSetProtocolError('GitHub runner list response is missing runners'); - } - runners.push(...response.runners); - if (response.runners.length < 100 || runners.length >= response.total_count) return runners; - } - throw new ScaleSetProtocolError(`GitHub runner inventory exceeded ${MAX_GITHUB_RUNNER_PAGES} pages`); - } - - /** Fetch one runner directly from GitHub immediately before a destructive action. */ - async getGitHubRunner(runnerId: number, options: ScaleSetRequestOptions = {}): Promise { - const url = githubApiUrl(this.config, `${this.runnerListPath()}/${runnerId}`); - const accessToken = await this.getAccessToken(); - const result = await executeRequest( - this.fetchImplementation, - url, - { - method: 'GET', - headers: { - Accept: 'application/vnd.github+json', - Authorization: `Bearer ${accessToken}`, - 'User-Agent': this.currentUserAgent, - 'X-GitHub-Api-Version': '2022-11-28', - }, - signal: options.signal, - }, - [200, 404], - ); - if (result.response.status === 404) return null; - return parseJsonResponse(result, 'GET', url); - } - - async listScaleSetRunnerStates( - runnerScaleSetId: number, - options: ScaleSetRequestOptions = {}, - ): Promise { - const [actionsRunners, githubRunners] = await Promise.all([ - this.listRunners(options), - this.listGitHubRunners(options), - ]); - const githubById = new Map(); - const duplicateIds = new Set(); - for (const runner of githubRunners) { - if (githubById.has(runner.id)) duplicateIds.add(runner.id); - else githubById.set(runner.id, runner); - } - return actionsRunners - .filter((runner) => runner.runnerScaleSetId === runnerScaleSetId) - .map((runner) => { - const githubRunner = duplicateIds.has(runner.id) ? undefined : githubById.get(runner.id); - const exact = githubRunner?.name === runner.name; - const status = - exact && (githubRunner.status === 'online' || githubRunner.status === 'offline') - ? githubRunner.status - : 'unknown'; - return { - runnerId: runner.id, - runnerName: runner.name, - scaleSetId: runner.runnerScaleSetId, - status, - busy: exact && typeof githubRunner.busy === 'boolean' ? githubRunner.busy : undefined, - }; - }); - } - async getRunnerByName(runnerName: string, options: ScaleSetRequestOptions = {}): Promise { const { result, url } = await this.actionsRequest('GET', RUNNER_ENDPOINT, { expectedStatuses: [200], @@ -605,17 +502,6 @@ export class GitHubActionsScaleSetClient { return accessToken; } - private runnerListPath(): string { - switch (this.config.scope) { - case 'organization': - return `/orgs/${this.config.organization}/actions/runners`; - case 'repository': - return `/repos/${this.config.organization}/${this.config.repository}/actions/runners`; - case 'enterprise': - return `/enterprises/${this.config.enterprise}/actions/runners`; - } - } - private async getActionsServiceAdminConnection( registrationToken: string, signal?: AbortSignal, diff --git a/lambdas/libs/github-actions-scale-set/src/types.ts b/lambdas/libs/github-actions-scale-set/src/types.ts index 4d800646ec..11d387bf4c 100644 --- a/lambdas/libs/github-actions-scale-set/src/types.ts +++ b/lambdas/libs/github-actions-scale-set/src/types.ts @@ -94,21 +94,6 @@ export interface RunnerReference { runnerScaleSetId: number; } -export interface GitHubRunnerReference { - id: number; - name: string; - status: 'online' | 'offline' | string; - busy: boolean; -} - -export interface ScaleSetRunnerState { - runnerId: number; - runnerName: string; - scaleSetId: number; - status: 'online' | 'offline' | 'unknown'; - busy: boolean | undefined; -} - export interface RunnerScaleSetJitRunnerConfig { runner: RunnerReference | null; encodedJITConfig: string; diff --git a/lambdas/services/scale-set/README.md b/lambdas/services/scale-set/README.md index 43758bb376..e42ada323d 100644 --- a/lambdas/services/scale-set/README.md +++ b/lambdas/services/scale-set/README.md @@ -71,39 +71,25 @@ GitHub App ID and private-key values are reloaded from SSM whenever an installat Runtime settings: -| Environment variable | Default | -| --------------------------------------------- | ------------ | -| `SCALE_SET_HEALTH_PORT` | `8080` | -| `SCALE_SET_HEALTH_STALE_AFTER_SECONDS` | `180` | -| `SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS` | `110` | -| `SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS` | `10` | -| `SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS` | `1` | -| `SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS` | `30` | -| `SCALE_SET_CONTROLLER_MODE` | `controller` | -| `SCALE_SET_JANITOR_INTERVAL_SECONDS` | `300` | - -Set `SCALE_SET_CONTROLLER_MODE=janitor` in a separate container or task. Janitor mode does not open a message -session and does not create a missing scale set. Each pass lists EC2 instances through the exact ownership tags, -joins the Actions-service runner identity with the public GitHub runner status, removes only an exact runner whose -fresh status is explicitly not busy, and terminates the matching EC2 instance only after GitHub removal succeeds. Busy, -unknown-status, mismatched, missing, and otherwise unknown runners are retained. Run this -process independently from the controller so it can clean idle capacity after a controller failure; do not run both -modes against the same scale set unless the janitor is deliberately being used as the recovery owner. +| Environment variable | Default | +| --------------------------------------------- | ------- | +| `SCALE_SET_HEALTH_PORT` | `8080` | +| `SCALE_SET_HEALTH_STALE_AFTER_SECONDS` | `180` | +| `SCALE_SET_SHUTDOWN_TIMEOUT_SECONDS` | `110` | +| `SCALE_SET_SESSION_CLOSE_TIMEOUT_SECONDS` | `10` | +| `SCALE_SET_RECONNECT_INITIAL_BACKOFF_SECONDS` | `1` | +| `SCALE_SET_RECONNECT_MAX_BACKOFF_SECONDS` | `30` | +| `LOG_LEVEL` | `info` | ## Reconciliation and health -Demand is calculated as `max(totalAssignedJobs, min(maxRunners, minRunners + totalAssignedJobs))`. The maximum therefore bounds requested idle capacity without ever requesting scale-down below work GitHub has already assigned. Job-started and job-completed messages maintain a bounded in-memory lifecycle cache. State is unknown after restart unless GitHub provides an exact runner match, and the compute provider must retain unknown or busy runners. +Demand is calculated as `max(totalAssignedJobs, min(maxRunners, minRunners + totalAssignedJobs))`. The maximum therefore bounds requested idle capacity without ever requesting scale-down below work GitHub has already assigned. Job-started and job-completed messages maintain a bounded in-memory lifecycle cache. After a restart, lifecycle state is unknown, but provider-owned EC2 tags still identify current capacity. The aggregate scale-set busy count protects unknown runners from scale-down until it reaches zero. Runner deletion executes inside the serialized reconcile loop and re-checks the exact Actions-service runner identity by name before removal. -The public GitHub runner inventory is not fetched on ordinary steady-state or scale-up polls. A compute provider explicitly requests one bounded, owner-scope inventory refresh when it needs to verify old handed-off capacity or perform safe scale-down; owner inventory is briefly shared across reconcilers. The first provider pass is marked lifecycle-only and the second is explicitly marked inventory-complete, so the provider cannot mistake a post-restart gap for an authoritative absence. Runner deletion executes inside the serialized reconcile loop, re-fetches the Actions identity by name, and then performs a fresh public GitHub lookup to verify the exact ID/name and confirm the runner is not busy before issuing the delete. +The public GitHub runner inventory is not fetched by the scale-set service. The selected compute provider reconciles its own capacity inventory, while the scale-set session remains the source of demand and aggregate busy-runner statistics. This keeps the service aligned with the upstream scale-set API and supports GitHub.com, GHES, and data-residency endpoints without relying on a separate public REST runner endpoint. Messages follow the upstream scale-set listener order: acknowledge first, then acquire available jobs, update lifecycle state, and reconcile compute. Provider failures therefore stop that reconciler after the message has been acknowledged; provider results expose one error outcome rather than the control-plane scaling retry classification. A typed busy/unknown retention remains a successful reconciliation. Session and transport failures are handled separately by bounded client retries or session recreation. -The EC2 provider counts a `config-published` instance as serving only during the orchestration request's boot window (`bootTimeoutMinutes`, default `10`) or after an exact online or `JobStarted` identity is observed. After the window, offline or unknown capacity is retained rather than terminated, and the complete inventory pass allows it to stop suppressing a replacement. Instances left in an earlier or unknown publication state are also retained for operator recovery and never terminated speculatively. EC2 ownership includes a SHA-256 hash of the canonical GitHub configuration scope, preventing the same runner-config name and numeric scale-set ID in another GitHub scope from colliding. A bounded one-instance physical surge may replace ambiguous capacity; once that ceiling is reached, the provider reports retained capacity instead of creating an unbounded replacement loop. - -The recovery janitor intentionally has a stricter destructive boundary than normal reconciliation: it only acts on -an exact EC2 ownership match plus an exact Actions runner ID/name match plus a fresh GitHub `busy: false` response. -It treats an unavailable GitHub inventory as unknown and leaves the instance untouched. The GitHub App therefore -needs organization `Self-hosted runners: Read & write` permission for the final status lookup and removal. +The EC2 provider uses the tagged, `config-published` EC2 instances as its capacity inventory. The GitHub Actions scale-set session supplies `desiredRunners` through `totalAssignedJobs` and the aggregate `totalBusyRunners` count. When capacity is above desired and the aggregate busy count is zero, tagged runners may be removed through the Actions service and their matching EC2 instances terminated; busy, contradictory, or unknown identities are retained. The provider uses the boot window (`bootTimeoutMinutes`, default `10`) for newly launched instances and keeps interrupted publication states from being counted as serving. EC2 ownership includes a SHA-256 hash of the canonical GitHub configuration scope, preventing the same runner-config name and numeric scale-set ID in another GitHub scope from colliding. No public GitHub REST runner inventory call is required. - `GET /healthz` reports controller liveness and is used by Docker/ECS. External GitHub outages remain live but degraded to avoid restart loops. - `GET /readyz` reports readiness and returns 503 unless every reconciler is ready. @@ -116,23 +102,6 @@ Build from the repository root: docker build --target runtime -f lambdas/services/scale-set/Dockerfile -t scale-set-controller . ``` -For local recovery, start the janitor as a separate container. The profile file must be mounted inside the container -and the profile's IAM identity must be allowed to describe and terminate only the tagged runner instances, plus read -the configured SSM parameters: - -```shell -docker run --rm --name scale-set-janitor --no-healthcheck \ - -e SCALE_SET_CONTROLLER_MODE=janitor \ - -e SCALE_SET_JANITOR_INTERVAL_SECONDS=300 \ - -e AWS_REGION=eu-west-1 \ - -e AWS_DEFAULT_REGION=eu-west-1 \ - -e AWS_PROFILE=forge-ops-dev \ - -e AWS_SDK_LOAD_CONFIG=1 \ - -v "$HOME/.aws:/home/node/.aws:ro" \ - -e SCALE_SET_CONTROLLER_MANIFEST="$( { sessionCloseTimeoutMs: 10000, reconnectInitialBackoffMs: 1000, reconnectMaxBackoffMs: 30000, - mode: 'controller', - janitorIntervalMs: 300000, }); }); @@ -62,19 +60,6 @@ describe('scale-set service configuration', () => { ).toThrow('must not exceed'); }); - it('supports an independent janitor mode with a bounded poll interval', () => { - expect( - parseScaleSetServiceConfig({ - SCALE_SET_CONTROLLER_MANIFEST: '{}', - SCALE_SET_CONTROLLER_MODE: 'janitor', - SCALE_SET_JANITOR_INTERVAL_SECONDS: '60', - }), - ).toMatchObject({ mode: 'janitor', janitorIntervalMs: 60000 }); - expect(() => - parseScaleSetServiceConfig({ SCALE_SET_CONTROLLER_MANIFEST: '{}', SCALE_SET_CONTROLLER_MODE: 'invalid' }), - ).toThrow('must be controller or janitor'); - }); - it('validates production selectors and numeric runtime settings', () => { expect(() => parseScaleSetServiceConfig({})).toThrow('provide exactly one'); expect(() => diff --git a/lambdas/services/scale-set/src/config.ts b/lambdas/services/scale-set/src/config.ts index d7377210fc..0f048dbe6c 100644 --- a/lambdas/services/scale-set/src/config.ts +++ b/lambdas/services/scale-set/src/config.ts @@ -54,8 +54,6 @@ export interface ScaleSetServiceConfig { sessionCloseTimeoutMs: number; reconnectInitialBackoffMs: number; reconnectMaxBackoffMs: number; - mode: 'controller' | 'janitor'; - janitorIntervalMs: number; } export type ScaleSetServiceEnvironment = Readonly>; @@ -69,7 +67,6 @@ const DEFAULT_SHUTDOWN_TIMEOUT_SECONDS = 110; const DEFAULT_SESSION_CLOSE_TIMEOUT_SECONDS = 10; const DEFAULT_RECONNECT_INITIAL_BACKOFF_SECONDS = 1; const DEFAULT_RECONNECT_MAX_BACKOFF_SECONDS = 30; -const DEFAULT_JANITOR_INTERVAL_SECONDS = 300; const MAX_RECONCILERS = 1000; const MAX_PROVIDER_CONFIG_NODES = 10_000; const MAX_PROVIDER_CONFIG_DEPTH = 32; @@ -140,11 +137,6 @@ export function parseScaleSetServiceConfig(environment: ScaleSetServiceEnvironme ); } - const mode = environment.SCALE_SET_CONTROLLER_MODE?.trim() || 'controller'; - if (mode !== 'controller' && mode !== 'janitor') { - throw new ScaleSetConfigurationError('SCALE_SET_CONTROLLER_MODE must be controller or janitor'); - } - return { ...(manifest ? { manifest } : {}), ...(groupConfigPath ? { groupConfigPath, groupName, groupRevision } : {}), @@ -173,13 +165,6 @@ export function parseScaleSetServiceConfig(environment: ScaleSetServiceEnvironme }) * 1000, reconnectInitialBackoffMs, reconnectMaxBackoffMs, - mode, - janitorIntervalMs: - parseInteger(environment, 'SCALE_SET_JANITOR_INTERVAL_SECONDS', { - defaultValue: DEFAULT_JANITOR_INTERVAL_SECONDS, - minimum: 10, - maximum: 86400, - }) * 1000, }; } diff --git a/lambdas/services/scale-set/src/controller.ts b/lambdas/services/scale-set/src/controller.ts index 6d67cb2a3b..5cbd2ac229 100644 --- a/lambdas/services/scale-set/src/controller.ts +++ b/lambdas/services/scale-set/src/controller.ts @@ -20,6 +20,10 @@ export class ScaleSetController { } async run(signal: AbortSignal): Promise { + this.controllerLogger.debug('scale_set_reconcilers_starting', { + reconcilerCount: this.manifest.reconcilers.length, + runnerConfigNames: this.manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); const completions = this.manifest.reconcilers.map(async (config) => { const status = this.health.reporter(config.runnerConfigName); try { @@ -39,14 +43,6 @@ export class ScaleSetController { this.health.markStopping(); await Promise.all(completions); } - - async recover(signal: AbortSignal): Promise { - await Promise.all( - this.manifest.reconcilers.map(async (config) => { - await new ScaleSetReconciler(config, this.serviceConfig, this.dependencies).recover(signal); - }), - ); - } } async function waitForAbort(signal: AbortSignal): Promise { diff --git a/lambdas/services/scale-set/src/logger.test.ts b/lambdas/services/scale-set/src/logger.test.ts index 6e7c881aa7..5e58b47d57 100644 --- a/lambdas/services/scale-set/src/logger.test.ts +++ b/lambdas/services/scale-set/src/logger.test.ts @@ -1,7 +1,21 @@ -import { logger, sanitizeLogAttributes } from './logger'; +import { createScaleSetLogger, logger, sanitizeLogAttributes } from './logger'; import { ScaleSetConfigurationError } from './config'; describe('redacted structured logging', () => { + it('emits debug records when LOG_LEVEL is debug', () => { + const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined); + createScaleSetLogger({ LOG_LEVEL: 'debug' }).debug('debug_event', { reconcilerCount: 2 }); + expect(spy).toHaveBeenCalledWith(expect.stringContaining('"event":"debug_event"')); + spy.mockRestore(); + }); + + it('does not emit debug records at the default info level', () => { + const spy = vi.spyOn(console, 'debug').mockImplementation(() => undefined); + createScaleSetLogger({}).debug('hidden_debug_event'); + expect(spy).not.toHaveBeenCalled(); + spy.mockRestore(); + }); + it('redacts nested secrets and strips log-injection characters', () => { expect( sanitizeLogAttributes({ diff --git a/lambdas/services/scale-set/src/logger.ts b/lambdas/services/scale-set/src/logger.ts index 188950c86a..639f8a2177 100644 --- a/lambdas/services/scale-set/src/logger.ts +++ b/lambdas/services/scale-set/src/logger.ts @@ -3,8 +3,12 @@ const SENSITIVE_KEY = /(authorization|credential|encodedjit|jitconfig|password|p const SAFE_ERROR_MESSAGE_NAMES = new Set(['ScaleSetConfigurationError']); const MAX_LOG_STRING_LENGTH = 1024; const MAX_LOG_DEPTH = 4; +const LOG_LEVEL_PRIORITY = { debug: 10, info: 20, warn: 30, error: 40 } as const; + +export type ScaleSetLogLevel = keyof typeof LOG_LEVEL_PRIORITY; export interface ScaleSetLogger { + debug(event: string, attributes?: Readonly>): void; info(event: string, attributes?: Readonly>): void; warn(event: string, attributes?: Readonly>): void; error(event: string, attributes?: Readonly>): void; @@ -45,7 +49,17 @@ export function sanitizeLogAttributes(attributes: Readonly; } -function write(level: 'info' | 'warn' | 'error', event: string, attributes?: Readonly>): void { +function parseLogLevel(value: string | undefined): ScaleSetLogLevel { + return value !== undefined && value in LOG_LEVEL_PRIORITY ? (value as ScaleSetLogLevel) : 'info'; +} + +function write( + level: ScaleSetLogLevel, + minimumLevel: ScaleSetLogLevel, + event: string, + attributes?: Readonly>, +): void { + if (LOG_LEVEL_PRIORITY[level] < LOG_LEVEL_PRIORITY[minimumLevel]) return; const record = JSON.stringify({ timestamp: new Date().toISOString(), level, @@ -54,11 +68,20 @@ function write(level: 'info' | 'warn' | 'error', event: string, attributes?: Rea }); if (level === 'error') console.error(record); else if (level === 'warn') console.warn(record); - else console.info(record); + else if (level === 'info') console.info(record); + else console.debug(record); +} + +export function createScaleSetLogger( + environment: Readonly> = process.env, +): ScaleSetLogger { + const minimumLevel = parseLogLevel(environment.LOG_LEVEL); + return { + debug: (event, attributes) => write('debug', minimumLevel, event, attributes), + info: (event, attributes) => write('info', minimumLevel, event, attributes), + warn: (event, attributes) => write('warn', minimumLevel, event, attributes), + error: (event, attributes) => write('error', minimumLevel, event, attributes), + }; } -export const logger: ScaleSetLogger = { - info: (event, attributes) => write('info', event, attributes), - warn: (event, attributes) => write('warn', event, attributes), - error: (event, attributes) => write('error', event, attributes), -}; +export const logger = createScaleSetLogger(); diff --git a/lambdas/services/scale-set/src/main.ts b/lambdas/services/scale-set/src/main.ts index da56014130..b97cb112e3 100644 --- a/lambdas/services/scale-set/src/main.ts +++ b/lambdas/services/scale-set/src/main.ts @@ -9,7 +9,7 @@ import { startScaleSetHealthServer, type ScaleSetHealthServer } from './health-s import { ScaleSetServiceRuntime } from './lifecycle'; import { logger } from './logger'; import { createDefaultControllerManifestLoader, defaultParameterStore } from './parameter-store'; -import { abortableSleep, TtlScaleSetRunnerInventoryCache, type ScaleSetReconcilerDependencies } from './reconciler'; +import { abortableSleep, type ScaleSetReconcilerDependencies } from './reconciler'; async function main(): Promise { logger.info('scale_set_controller_configuration_loading', { @@ -24,6 +24,11 @@ async function main(): Promise { reconcilerCount: manifest.reconcilers.length, runnerConfigNames: manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), }); + logger.debug('scale_set_controller_reconcilers_loaded', { + groupName: manifest.groupName, + reconcilerCount: manifest.reconcilers.length, + runnerConfigNames: manifest.reconcilers.map(({ runnerConfigName }) => runnerConfigName), + }); const computeProviders = createScaleSetComputeProviderRegistry(); const githubHttp = createScaleSetGitHubHttp(); const dependencies: ScaleSetReconcilerDependencies = { @@ -54,16 +59,8 @@ async function main(): Promise { sleep: abortableSleep, random: Math.random, closeSignal: AbortSignal.timeout, - runnerInventory: new TtlScaleSetRunnerInventoryCache(), }; const controller = new ScaleSetController(manifest, serviceConfig, dependencies, logger); - - if (serviceConfig.mode === 'janitor') { - await runJanitor(controller, manifest.groupName, manifest.revision, serviceConfig.janitorIntervalMs); - await githubHttp.close(); - return; - } - const runtime = new ScaleSetServiceRuntime(serviceConfig, controller); let healthServer: ScaleSetHealthServer | undefined; @@ -97,35 +94,6 @@ async function main(): Promise { } } -async function runJanitor( - controller: ScaleSetController, - groupName: string, - revision: string | undefined, - intervalMs: number, -): Promise { - const abortController = new AbortController(); - const shutdown = (signal: NodeJS.Signals) => { - logger.info('scale_set_janitor_shutdown_requested', { signal, groupName }); - abortController.abort(new Error(`received ${signal}`)); - }; - const onSigterm = () => shutdown('SIGTERM'); - const onSigint = () => shutdown('SIGINT'); - process.once('SIGTERM', onSigterm); - process.once('SIGINT', onSigint); - - try { - logger.info('scale_set_janitor_started', { groupName, revision, intervalMs }); - while (!abortController.signal.aborted) { - await controller.recover(abortController.signal); - await abortableSleep(intervalMs, abortController.signal); - } - } finally { - process.removeListener('SIGTERM', onSigterm); - process.removeListener('SIGINT', onSigint); - logger.info('scale_set_janitor_stopped', { groupName }); - } -} - void main().catch((error) => { logger.error('scale_set_controller_fatal_failure', { error }); process.exitCode = 1; diff --git a/lambdas/services/scale-set/src/reconciler.test.ts b/lambdas/services/scale-set/src/reconciler.test.ts index c0db5fb853..e9de83afaf 100644 --- a/lambdas/services/scale-set/src/reconciler.test.ts +++ b/lambdas/services/scale-set/src/reconciler.test.ts @@ -1,15 +1,11 @@ -import { - ScaleSetHttpError, - type MessageSessionClient, - type RunnerScaleSetMessage, -} from '@aws-github-runner/github-actions-scale-set'; +import type { MessageSessionClient, RunnerScaleSetMessage } from '@aws-github-runner/github-actions-scale-set'; import type { ScaleSetComputeProvider, ScaleSetReconcileResult } from '@aws-github-runner/compute-providers/scale-set'; +import { describe, expect, it, vi } from 'vitest'; import type { ScaleSetReconcilerConfig, ScaleSetServiceConfig } from './config'; import type { ScaleSetReconcilerStatusReporter } from './health'; import { ScaleSetReconciler, - TtlScaleSetRunnerInventoryCache, calculateDesiredRunners, validateProviderResult, type ScaleSetReconcilerClient, @@ -46,7 +42,6 @@ function result(overrides: Partial = {}): ScaleSetRecon status: 'converged', desiredRunners: 1, currentRunners: 1, - needsRunnerInventory: false, actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0 }, errors: [], ...overrides, @@ -103,10 +98,7 @@ function fixture(options: { }), createMessageSessionClient: vi.fn().mockResolvedValue(options.session as MessageSessionClient), generateJitRunnerConfig: vi.fn(), - getGitHubRunner: vi.fn().mockResolvedValue({ id: 5, name: 'runner-5', status: 'online', busy: false }), getRunnerByName: vi.fn(), - listGitHubRunners: vi.fn().mockResolvedValue([]), - listRunners: vi.fn().mockResolvedValue([]), removeRunner: vi.fn(), systemInfo: { scaleSetId: 42 }, setSystemInfo: vi.fn(), @@ -115,14 +107,13 @@ function fixture(options: { createAccessTokenProvider: vi.fn().mockResolvedValue(async () => ({ token: 'not-a-real-token' })), createClient: vi.fn().mockReturnValue(client), computeProviders: { create: vi.fn().mockReturnValue(computeProvider) }, - logger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() }, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }, sleep: vi.fn(async (_delay, signal) => { if (!signal.aborted) await new Promise((resolve) => signal.addEventListener('abort', () => resolve(), { once: true })); }), random: () => 0, closeSignal: () => new AbortController().signal, - runnerInventory: new TtlScaleSetRunnerInventoryCache(), parameterStore: { get: vi.fn().mockResolvedValue(new Map()), put: vi.fn() }, }; return { client, computeProvider, dependencies }; @@ -169,7 +160,7 @@ describe('ScaleSetReconciler', () => { ); expect(dependencies.logger.info).toHaveBeenCalledWith( 'scale_set_compute_provider_reconcile_started', - expect.objectContaining({ computeProviderType: 'ec2', desiredRunners: 1, runnerInventoryComplete: false }), + expect.objectContaining({ computeProviderType: 'ec2', desiredRunners: 1, busyRunners: 0 }), ); }); @@ -208,34 +199,6 @@ describe('ScaleSetReconciler', () => { ); }); - it('runs recovery without opening a session or registering a missing scale set', async () => { - const abort = new AbortController(); - const session = { - session: { statistics: undefined }, - getMessage: vi.fn(), - close: vi.fn(), - }; - const reconcile = vi.fn().mockResolvedValue(result({ desiredRunners: 0, currentRunners: 0 })); - const { client, computeProvider, dependencies } = fixture({ session, reconcile }); - vi.mocked(client.getRunnerScaleSetById).mockResolvedValue({ id: 42, name: 'linux' }); - - await new ScaleSetReconciler(config, serviceConfig, dependencies).recover(abort.signal); - - expect(client.createMessageSessionClient).not.toHaveBeenCalled(); - expect(client.createRunnerScaleSet).not.toHaveBeenCalled(); - expect(computeProvider.reconcile).toHaveBeenCalledWith( - expect.objectContaining({ - desiredRunners: 0, - recoveryOnly: true, - runnerInventoryComplete: true, - }), - ); - expect(dependencies.logger.info).toHaveBeenCalledWith( - 'scale_set_recovery_reconciled', - expect.objectContaining({ actions: expect.any(Object) }), - ); - }); - it('acknowledges before acquisition, lifecycle observation, and reconciliation', async () => { const order: string[] = []; const abort = new AbortController(); @@ -253,7 +216,7 @@ describe('ScaleSetReconciler', () => { }; const reconcile = vi.fn(async (request) => { order.push('reconcile'); - expect(request.runnerInventoryComplete).toBe(false); + expect(request.busyRunners).toBe(0); expect(request.bootTimeoutMinutes).toBe(10); expect(request.runnerStates).toContainEqual( expect.objectContaining({ runnerId: 5, runnerName: 'runner-5', lifecycle: 'started' }), @@ -273,132 +236,6 @@ describe('ScaleSetReconciler', () => { }); }); - it('does not query public or Actions runner inventory on steady-state empty polls', async () => { - const abort = new AbortController(); - let calls = 0; - const reconcile = vi.fn(async () => { - calls += 1; - if (calls === 2) abort.abort(); - return result({ desiredRunners: 0, currentRunners: 0 }); - }); - const session = { - session: { statistics: { ...message().statistics, totalAssignedJobs: 0 } }, - getMessage: vi.fn().mockResolvedValue(null), - close: vi.fn(), - }; - const { client, dependencies } = fixture({ session, reconcile }); - await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); - expect(reconcile).toHaveBeenCalledTimes(2); - expect(client.listGitHubRunners).not.toHaveBeenCalled(); - expect(client.listRunners).not.toHaveBeenCalled(); - }); - - it('performs the typed inventory second pass whenever requested, including at desired physical capacity', async () => { - const abort = new AbortController(); - const reconcile = vi - .fn() - .mockResolvedValueOnce( - result({ - status: 'retained', - desiredRunners: 1, - currentRunners: 1, - needsRunnerInventory: true, - actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, - errors: [], - }), - ) - .mockImplementationOnce(async (request) => { - expect(request.runnerInventoryComplete).toBe(true); - expect(request.runnerStates).toContainEqual({ - runnerId: 5, - runnerName: 'runner-5', - scaleSetId: 42, - status: 'online', - busy: false, - lifecycle: 'unknown', - }); - abort.abort(); - return result({ desiredRunners: 1, currentRunners: 1 }); - }); - const session = { - session: { statistics: message().statistics }, - close: vi.fn(), - }; - const { client, dependencies } = fixture({ session, reconcile }); - vi.mocked(client.listRunners).mockResolvedValue([{ id: 5, name: 'runner-5', runnerScaleSetId: 42 }]); - vi.mocked(client.listGitHubRunners).mockResolvedValue([{ id: 5, name: 'runner-5', status: 'online', busy: false }]); - await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); - expect(reconcile.mock.calls[0]?.[0]).toEqual(expect.objectContaining({ runnerInventoryComplete: false })); - expect(client.listGitHubRunners).toHaveBeenCalledTimes(1); - expect(client.listRunners).toHaveBeenCalledTimes(1); - }); - - it('retains capacity and retries after a runner inventory 404 during recovery', async () => { - const abort = new AbortController(); - const inventoryError = new ScaleSetHttpError({ - method: 'GET', - url: 'https://api.github.com/orgs/example/actions/runners', - status: 404, - statusText: 'Not Found', - headers: new Headers(), - responseBody: '', - }); - const reconcile = vi.fn().mockResolvedValue( - result({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: true, - actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, - }), - ); - const session = { - session: { statistics: message().statistics }, - getMessage: vi.fn(async () => { - abort.abort(); - throw new DOMException('aborted', 'AbortError'); - }), - close: vi.fn(), - }; - const { client, dependencies } = fixture({ session, reconcile }); - vi.mocked(client.listGitHubRunners).mockRejectedValue(inventoryError); - const status = reporter(); - - await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); - - expect(status.markFailed).not.toHaveBeenCalled(); - expect(reconcile).toHaveBeenCalledTimes(1); - expect(dependencies.logger.warn).toHaveBeenCalledWith( - 'scale_set_runner_inventory_unavailable', - expect.objectContaining({ requestMethod: 'GET', requestStatus: 404, requestCode: 'NOT_FOUND' }), - ); - expect(client.listGitHubRunners).toHaveBeenCalledTimes(1); - }); - - it('rejects a provider that requests another inventory after the complete second pass', async () => { - const abort = new AbortController(); - const reconcile = vi.fn().mockResolvedValue( - result({ - status: 'retained', - currentRunners: 1, - needsRunnerInventory: true, - actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 1 }, - }), - ); - const session = { session: { statistics: message().statistics }, close: vi.fn() }; - const { dependencies } = fixture({ session, reconcile }); - dependencies.sleep = vi.fn(async () => abort.abort()); - const status = reporter(); - - await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, status); - - expect(reconcile).toHaveBeenCalledTimes(2); - expect(status.markFailed).toHaveBeenCalledWith( - expect.objectContaining({ - message: 'scale-set compute provider requested inventory after a complete inventory pass', - }), - ); - }); - it('acknowledges and stops when reconciliation rejects', async () => { const abort = new AbortController(); const session = { @@ -476,7 +313,7 @@ describe('ScaleSetReconciler', () => { expect(dependencies.sleep).not.toHaveBeenCalled(); }); - it('does not request inventory after a provider error result', async () => { + it('does not call public GitHub runner inventory after a provider error result', async () => { const session = { session: { statistics: undefined }, getMessage: vi.fn().mockResolvedValue(message()), @@ -488,24 +325,21 @@ describe('ScaleSetReconciler', () => { result({ status: 'error', currentRunners: 0, - needsRunnerInventory: true, errors: [{ operation: 'launch', code: 'EC2_LAUNCH_FAILED' }], }), ); - const { client, dependencies } = fixture({ session, reconcile }); + const { dependencies } = fixture({ session, reconcile }); const status = reporter(); await new ScaleSetReconciler(config, serviceConfig, dependencies).run(new AbortController().signal, status); expect(reconcile).toHaveBeenCalledOnce(); - expect(client.listGitHubRunners).not.toHaveBeenCalled(); - expect(client.listRunners).not.toHaveBeenCalled(); expect(status.markFailed).toHaveBeenCalledWith( expect.objectContaining({ name: 'ScaleSetProviderReconciliationError' }), ); }); - it('re-fetches exact state in the serialized loop and acknowledges a typed busy retention', async () => { + it('uses the Actions-service identity check without public runner verification', async () => { const abort = new AbortController(); const order: string[] = []; const session = { @@ -523,7 +357,7 @@ describe('ScaleSetReconciler', () => { const reconcile = vi.fn(async (request) => { order.push('reconcile'); await expect(request.removeRunner({ runnerId: 5, runnerName: 'runner-5', scaleSetId: 42 })).resolves.toEqual({ - status: 'retained_busy', + status: 'removed', }); abort.abort(); return result({ @@ -537,14 +371,10 @@ describe('ScaleSetReconciler', () => { order.push('actions-refetch'); return { id: 5, name: 'runner-5', runnerScaleSetId: 42 }; }); - vi.mocked(client.getGitHubRunner).mockImplementation(async () => { - order.push('github-refetch'); - return { id: 5, name: 'runner-5', status: 'online', busy: true }; - }); await new ScaleSetReconciler(config, serviceConfig, dependencies).run(abort.signal, reporter()); - expect(order).toEqual(['delete', 'acquire', 'reconcile', 'actions-refetch', 'github-refetch']); - expect(client.removeRunner).not.toHaveBeenCalled(); + expect(order).toEqual(['delete', 'acquire', 'reconcile', 'actions-refetch']); + expect(client.removeRunner).toHaveBeenCalledWith(5, { signal: abort.signal }); expect(session.deleteMessage).toHaveBeenCalledOnce(); expect(session.acquireJobs).toHaveBeenCalledTimes(1); }); @@ -570,24 +400,11 @@ describe('reconciler helpers', () => { expect(() => calculateDesiredRunners(-1, 0, 1)).toThrow('non-negative integer'); }); - it('shares successful inventory loads and retries failed loads', async () => { - let now = 0; - const cache = new TtlScaleSetRunnerInventoryCache(100, () => now); - const loader = vi.fn().mockResolvedValue([{ id: 1, name: 'a', status: 'online', busy: false }]); - await Promise.all([cache.get('scope', loader), cache.get('scope', loader)]); - expect(loader).toHaveBeenCalledTimes(1); - now = 101; - await cache.get('scope', loader); - expect(loader).toHaveBeenCalledTimes(2); - await expect(cache.get('failed', vi.fn().mockRejectedValue(new Error('nope')))).rejects.toThrow('nope'); - }); - it.each([ { status: 'unexpected' }, { status: 'retryable_error' }, { status: 'non_retryable_error' }, { retryable: true }, - { needsRunnerInventory: 'yes' }, { actions: { launched: 0, terminated: 0, retainedBusy: -1, retainedUnknown: 0 } }, { actions: { launched: 0, terminated: 0, retainedBusy: 0, retainedUnknown: 0, retryable: true } }, { status: 'converged', errors: [{ operation: 'list', code: 'UNEXPECTED_ERROR' }] }, @@ -595,7 +412,6 @@ describe('reconciler helpers', () => { { status: 'error', errors: [] }, { currentRunners: 0 }, { currentRunners: 2 }, - { needsRunnerInventory: true }, { status: 'retained' }, { errors: [{ operation: 'shell', code: 'BAD' }] }, { errors: [{ operation: 'list', code: 'contains spaces' }] }, diff --git a/lambdas/services/scale-set/src/reconciler.ts b/lambdas/services/scale-set/src/reconciler.ts index ff46d9806a..69c56e9cda 100644 --- a/lambdas/services/scale-set/src/reconciler.ts +++ b/lambdas/services/scale-set/src/reconciler.ts @@ -3,11 +3,9 @@ import { isScaleSetHttpError, ScaleSetProtocolError, type AccessToken, - type GitHubRunnerReference, type MessageSessionClient, type RunnerScaleSetMessage, type RunnerScaleSetStatistic, - type ScaleSetRunnerState as GitHubScaleSetRunnerState, } from '@aws-github-runner/github-actions-scale-set'; import type { ScaleSetComputeProvider, @@ -24,7 +22,6 @@ import type { ScaleSetReconcilerStatusReporter } from './health'; import type { ScaleSetLogger } from './logger'; const MAX_JIT_CONFIGURATION_BYTES = 1024 * 1024; -const SCALE_SET_INVENTORY_TTL_MS = 60_000; export interface ScaleSetComputeProviderFactory { create(type: string, input: ScaleSetComputeProviderFactoryInput): ScaleSetComputeProvider; @@ -34,13 +31,10 @@ export type ScaleSetReconcilerClient = Pick< GitHubActionsScaleSetClient, | 'createMessageSessionClient' | 'generateJitRunnerConfig' - | 'getGitHubRunner' | 'getRunnerGroupByName' | 'getRunnerScaleSet' | 'getRunnerScaleSetById' | 'getRunnerByName' - | 'listGitHubRunners' - | 'listRunners' | 'removeRunner' | 'createRunnerScaleSet' | 'setSystemInfo' @@ -55,39 +49,9 @@ export interface ScaleSetReconcilerDependencies { sleep(delayMs: number, signal: AbortSignal): Promise; random(): number; closeSignal(timeoutMs: number): AbortSignal; - runnerInventory: ScaleSetRunnerInventoryCache; parameterStore: ParameterStore; } -export interface ScaleSetRunnerInventoryCache { - get(key: string, loader: () => Promise): Promise; -} - -export class TtlScaleSetRunnerInventoryCache implements ScaleSetRunnerInventoryCache { - private readonly entries = new Map }>(); - - constructor( - private readonly ttlMs = 60_000, - private readonly now: () => number = Date.now, - ) {} - - async get( - key: string, - loader: () => Promise, - ): Promise { - const current = this.entries.get(key); - if (current !== undefined && current.expiresAt > this.now()) return await current.value; - const value = loader(); - this.entries.set(key, { expiresAt: this.now() + this.ttlMs, value }); - try { - return await value; - } catch (error) { - if (this.entries.get(key)?.value === value) this.entries.delete(key); - throw error; - } - } -} - interface LifecycleObservation { runnerId: number; runnerName: string; @@ -113,7 +77,6 @@ export class ScaleSetProviderReconciliationError extends Error { export class ScaleSetReconciler { private readonly lifecycle = new Map(); private readonly lifecycleLimit: number; - private inventory?: { expiresAt: number; value: Promise }; private resolvedScaleSetId?: number; private resolvedRunnerGroupId?: number; @@ -134,10 +97,13 @@ export class ScaleSetReconciler { try { const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); client = this.dependencies.createClient(this.config, accessTokenProvider); - const resolved = await this.resolveScaleSet(client, signal, true); + const resolved = await this.resolveScaleSet(client, signal); this.resolvedScaleSetId = resolved.scaleSetId; this.resolvedRunnerGroupId = resolved.runnerGroupId; client.setSystemInfo({ ...client.systemInfo, scaleSetId: resolved.scaleSetId }); + this.log('debug', 'scale_set_compute_provider_loading', { + computeProviderType: this.config.computeProvider.type, + }); provider = this.dependencies.computeProviders.create(this.config.computeProvider.type, { runnerConfigName: this.config.runnerConfigName, scaleSetId: resolved.scaleSetId, @@ -147,6 +113,9 @@ export class ScaleSetReconciler { this.log('info', 'scale_set_compute_provider_created', { computeProviderType: this.config.computeProvider.type, }); + this.log('debug', 'scale_set_compute_provider_loaded', { + computeProviderType: this.config.computeProvider.type, + }); } catch (error) { status.markFailed(error); this.log('error', 'scale_set_reconciler_initialization_failed', { @@ -174,6 +143,10 @@ export class ScaleSetReconciler { session = await client.createMessageSessionClient(this.scaleSetId, this.config.sessionOwner, { signal }); status.markSessionReady(); this.log('info', 'scale_set_session_created'); + this.log('debug', 'scale_set_session_scale_set_loaded', { + scaleSetName: session.session.runnerScaleSet?.name, + scaleSetLabels: session.session.runnerScaleSet?.labels?.map(({ name, type }) => ({ name, type })), + }); let latestStatistics = session.session.statistics ?? undefined; let lastMessageId = 0; if (latestStatistics !== undefined) { @@ -239,82 +212,9 @@ export class ScaleSetReconciler { status.markStopping(); } - /** Run one independent recovery pass without opening a message session or creating a scale set. */ - async recover(signal: AbortSignal): Promise { - let provider: ScaleSetComputeProvider; - let client: ScaleSetReconcilerClient; - try { - const accessTokenProvider = await this.dependencies.createAccessTokenProvider(this.config); - client = this.dependencies.createClient(this.config, accessTokenProvider); - const resolved = await this.resolveScaleSet(client, signal, false); - this.resolvedScaleSetId = resolved.scaleSetId; - this.resolvedRunnerGroupId = resolved.runnerGroupId; - client.setSystemInfo({ ...client.systemInfo, scaleSetId: resolved.scaleSetId }); - provider = this.dependencies.computeProviders.create(this.config.computeProvider.type, { - runnerConfigName: this.config.runnerConfigName, - scaleSetId: resolved.scaleSetId, - githubScope: this.config.githubConfigUrl, - configuration: this.config.computeProvider.configuration, - }); - } catch (error) { - if (signal.aborted) return; - this.log('warn', 'scale_set_recovery_initialization_failed', { - computeProviderType: this.config.computeProvider.type, - ...httpErrorLogAttributes(error), - error, - }); - return; - } - - try { - this.log('info', 'scale_set_runner_inventory_loading', { - githubConfigUrl: this.config.githubConfigUrl, - githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', - inventorySources: ['actions_service', 'github_rest'], - }); - const inventory = await this.loadScaleSetInventory(client, signal); - this.log('info', 'scale_set_runner_inventory_loaded', { - githubConfigUrl: this.config.githubConfigUrl, - runnerCount: inventory.length, - }); - const result = await this.reconcileProvider(provider, { - desiredRunners: 0, - recoveryOnly: true, - bootTimeoutMinutes: this.config.bootTimeoutMinutes, - runnerInventoryComplete: true, - runnerStates: this.mergeLifecycle(inventory), - ...this.createReconcileCallbacks(client, signal), - }); - validateProviderResult(result, 0); - throwIfProviderError(result); - this.log('info', 'scale_set_recovery_reconciled', { - computeProviderType: this.config.computeProvider.type, - currentRunners: result.currentRunners, - status: result.status, - actions: result.actions, - errorCount: result.errors.length, - }); - } catch (error) { - if (signal.aborted) return; - this.log('warn', 'scale_set_recovery_failed', { - failureStage: 'github_runner_inventory_or_ec2_recovery', - githubConfigUrl: this.config.githubConfigUrl, - githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', - diagnosis: - isScaleSetHttpError(error) && error.status === 404 - ? 'github_endpoint_not_found_or_app_not_authorized' - : undefined, - computeProviderType: this.config.computeProvider.type, - ...httpErrorLogAttributes(error), - error, - }); - } - } - private async resolveScaleSet( client: ScaleSetReconcilerClient, signal: AbortSignal, - registerMissing: boolean, ): Promise<{ scaleSetId: number; runnerGroupId?: number }> { let runnerGroupId = this.config.expectedRunnerGroupId; if (this.config.runnerGroupName !== undefined) { @@ -346,12 +246,7 @@ export class ScaleSetReconciler { this.config.scaleSetId === undefined ? await client.getRunnerScaleSet(runnerGroupId as number, this.config.scaleSetName, { signal }) : await client.getRunnerScaleSetById(this.config.scaleSetId, { signal }); - if ( - registerMissing && - configuredScaleSet === null && - runnerGroupId !== undefined && - this.config.scaleSetId === undefined - ) { + if (configuredScaleSet === null && runnerGroupId !== undefined && this.config.scaleSetId === undefined) { this.log('info', 'scale_set_registering', { runnerConfigName: this.config.runnerConfigName, scaleSetName: this.config.scaleSetName, @@ -391,6 +286,10 @@ export class ScaleSetReconciler { scaleSetId: configuredScaleSet.id, runnerGroupId, }); + this.log('debug', 'scale_set_labels_resolved', { + scaleSetName: configuredScaleSet.name, + scaleSetLabels: configuredScaleSet.labels?.map(({ name, type }) => ({ name, type })), + }); return { scaleSetId: configuredScaleSet.id, runnerGroupId }; } @@ -433,56 +332,15 @@ export class ScaleSetReconciler { this.config.maxRunners, ); const callbacks = this.createReconcileCallbacks(client, signal); - let result = await this.reconcileProvider(provider, { + const result = await this.reconcileProvider(provider, { desiredRunners, + busyRunners: statistics.totalBusyRunners, bootTimeoutMinutes: this.config.bootTimeoutMinutes, - runnerInventoryComplete: false, runnerStates: this.lifecycleStates(), ...callbacks, }); validateProviderResult(result, desiredRunners); throwIfProviderError(result); - if (result.needsRunnerInventory) { - let inventory: readonly GitHubScaleSetRunnerState[]; - try { - this.log('info', 'scale_set_runner_inventory_loading', { - githubConfigUrl: this.config.githubConfigUrl, - githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', - inventorySources: ['actions_service', 'github_rest'], - }); - inventory = await this.loadScaleSetInventory(client, signal); - this.log('info', 'scale_set_runner_inventory_loaded', { - githubConfigUrl: this.config.githubConfigUrl, - runnerCount: inventory.length, - }); - } catch (error) { - if (!isScaleSetHttpError(error) || error.status !== 404) throw error; - this.log('warn', 'scale_set_runner_inventory_unavailable', { - failureStage: 'github_runner_inventory', - githubConfigUrl: this.config.githubConfigUrl, - githubApiMode: this.config.forceGhes ? 'ghes' : 'hosted', - diagnosis: 'github_endpoint_not_found_or_app_not_authorized', - ...httpErrorLogAttributes(error), - error, - }); - this.logReconciliationResult(result, desiredRunners); - return; - } - result = await this.reconcileProvider(provider, { - desiredRunners, - bootTimeoutMinutes: this.config.bootTimeoutMinutes, - runnerInventoryComplete: true, - runnerStates: this.mergeLifecycle(inventory), - ...callbacks, - }); - validateProviderResult(result, desiredRunners); - throwIfProviderError(result); - if (result.needsRunnerInventory) { - throw new ScaleSetProtocolError( - 'scale-set compute provider requested inventory after a complete inventory pass', - ); - } - } this.logReconciliationResult(result, desiredRunners); } @@ -541,18 +399,9 @@ export class ScaleSetReconciler { ) { return { status: 'retained_unknown' as const }; } - const githubRunner = await client.getGitHubRunner(expected.runnerId, { signal: callbackSignal }); - if (githubRunner === null) return { status: 'retained_unknown' as const }; - if (githubRunner.id !== expected.runnerId || githubRunner.name !== expected.runnerName) { - return { status: 'retained_unknown' as const }; - } - if ( - typeof githubRunner.busy !== 'boolean' || - (githubRunner.status !== 'online' && githubRunner.status !== 'offline') - ) { - return { status: 'retained_unknown' as const }; - } - if (githubRunner.busy) return { status: 'retained_busy' as const }; + // Busy state comes from the aggregate scale-set statistics. The + // Actions-service runner reference above remains the exact identity + // check; no public GitHub REST runner call is required. try { await client.removeRunner(runner.id, { signal: callbackSignal }); } catch (error) { @@ -591,8 +440,7 @@ export class ScaleSetReconciler { this.log('info', 'scale_set_compute_provider_reconcile_started', { computeProviderType: this.config.computeProvider.type, desiredRunners: request.desiredRunners, - recoveryOnly: request.recoveryOnly ?? false, - runnerInventoryComplete: request.runnerInventoryComplete, + busyRunners: request.busyRunners, }); try { return await provider.reconcile(request); @@ -601,8 +449,7 @@ export class ScaleSetReconciler { this.log('error', 'scale_set_compute_provider_reconcile_failed', { computeProviderType: this.config.computeProvider.type, desiredRunners: request.desiredRunners, - recoveryOnly: request.recoveryOnly ?? false, - runnerInventoryComplete: request.runnerInventoryComplete, + busyRunners: request.busyRunners, error, }); throw new ScaleSetProviderReconciliationError(undefined, { cause: error }); @@ -641,25 +488,6 @@ export class ScaleSetReconciler { } } - private mergeLifecycle(inventory: readonly GitHubScaleSetRunnerState[]): ScaleSetRunnerState[] { - const result = inventory.map((runner): ScaleSetRunnerState => { - const observation = this.lifecycle.get(runner.runnerName); - const lifecycle = - observation !== undefined && - observation.runnerId === runner.runnerId && - observation.scaleSetId === runner.scaleSetId - ? observation.lifecycle - : 'unknown'; - return { ...runner, lifecycle }; - }); - const identities = new Set(result.map((runner) => `${runner.runnerId}\u0000${runner.runnerName}`)); - for (const observation of this.lifecycle.values()) { - if (identities.has(`${observation.runnerId}\u0000${observation.runnerName}`)) continue; - result.push({ ...observation, status: 'unknown', busy: undefined }); - } - return result; - } - private lifecycleStates(): ScaleSetRunnerState[] { return [...this.lifecycle.values()].map((observation) => ({ ...observation, @@ -668,37 +496,6 @@ export class ScaleSetReconciler { })); } - private inventoryCacheKey(): string { - const app = this.config.githubApp; - return [ - this.config.githubConfigUrl, - app.appIdParameterName, - app.installationIdParameterName, - app.privateKeyParameterName, - ].join('\u0000'); - } - - private async loadScaleSetInventory( - client: ScaleSetReconcilerClient, - signal: AbortSignal, - ): Promise { - if (this.inventory !== undefined && this.inventory.expiresAt > Date.now()) return await this.inventory.value; - const value = Promise.all([ - client.listRunners({ signal }), - this.dependencies.runnerInventory.get( - this.inventoryCacheKey(), - async () => await client.listGitHubRunners({ signal }), - ), - ]).then(([actionsRunners, githubRunners]) => joinRunnerInventory(actionsRunners, githubRunners, this.scaleSetId)); - this.inventory = { expiresAt: Date.now() + SCALE_SET_INVENTORY_TTL_MS, value }; - try { - return await value; - } catch (error) { - if (this.inventory?.value === value) this.inventory = undefined; - throw error; - } - } - private async closeSession(session: Pick): Promise { try { await session.close({ signal: this.dependencies.closeSignal(this.serviceConfig.sessionCloseTimeoutMs) }); @@ -707,7 +504,11 @@ export class ScaleSetReconciler { } } - private log(level: 'info' | 'warn' | 'error', event: string, attributes: Record = {}): void { + private log( + level: 'debug' | 'info' | 'warn' | 'error', + event: string, + attributes: Record = {}, + ): void { this.dependencies.logger[level](event, { groupRunnerConfig: this.config.runnerConfigName, scaleSetId: this.resolvedScaleSetId, @@ -760,14 +561,7 @@ export function validateProviderResult(result: ScaleSetReconcileResult, desiredR throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); } const record = value as Record; - const resultFields = new Set([ - 'status', - 'desiredRunners', - 'currentRunners', - 'needsRunnerInventory', - 'actions', - 'errors', - ]); + const resultFields = new Set(['status', 'desiredRunners', 'currentRunners', 'actions', 'errors']); if (Object.keys(record).some((key) => !resultFields.has(key))) { throw new ScaleSetProtocolError('scale-set compute provider returned an invalid reconciliation result'); } @@ -778,9 +572,6 @@ export function validateProviderResult(result: ScaleSetReconcileResult, desiredR if (record.desiredRunners !== desiredRunners || !boundedCount(record.currentRunners)) { throw new ScaleSetProtocolError('scale-set compute provider returned invalid capacity counts'); } - if (typeof record.needsRunnerInventory !== 'boolean') { - throw new ScaleSetProtocolError('scale-set compute provider returned an invalid inventory signal'); - } const actions = record.actions; if (typeof actions !== 'object' || actions === null || Array.isArray(actions)) { throw new ScaleSetProtocolError('scale-set compute provider returned invalid actions'); @@ -830,7 +621,7 @@ export function validateProviderResult(result: ScaleSetReconcileResult, desiredR let expectedStatus: ScaleSetReconcileResult['status'] = 'converged'; if (record.errors.length > 0 || (record.currentRunners as number) < desiredRunners) { expectedStatus = 'error'; - } else if (record.needsRunnerInventory || (record.currentRunners as number) > desiredRunners) { + } else if ((record.currentRunners as number) > desiredRunners) { expectedStatus = 'retained'; } if (record.status !== expectedStatus) { @@ -875,32 +666,3 @@ function httpErrorLogAttributes(error: unknown): Record { requestCode: error.code, }; } - -function joinRunnerInventory( - actionsRunners: readonly { id: number; name: string; runnerScaleSetId: number }[], - githubRunners: readonly GitHubRunnerReference[], - scaleSetId: number, -): GitHubScaleSetRunnerState[] { - const githubById = new Map(); - const duplicateIds = new Set(); - for (const runner of githubRunners) { - if (githubById.has(runner.id)) duplicateIds.add(runner.id); - else githubById.set(runner.id, runner); - } - return actionsRunners - .filter((runner) => runner.runnerScaleSetId === scaleSetId) - .map((runner) => { - const githubRunner = duplicateIds.has(runner.id) ? undefined : githubById.get(runner.id); - const exact = githubRunner?.name === runner.name; - return { - runnerId: runner.id, - runnerName: runner.name, - scaleSetId: runner.runnerScaleSetId, - status: - exact && (githubRunner.status === 'online' || githubRunner.status === 'offline') - ? githubRunner.status - : 'unknown', - busy: exact && typeof githubRunner.busy === 'boolean' ? githubRunner.busy : undefined, - }; - }); -}